{"meta":{"date":"2026-10-08","timezone":"UTC","generated_at":"2026-10-14T06:03:12Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":91,"failed_runs":66,"runs_with_occurrences":66,"runs_with_failure_pattern_matches":60,"runs_with_unmatched_occurrences":6,"runs_without_occurrence_rows":25,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108332744864960512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7437,"pr_state":"closed","pr_sha":"e7d1eb926779b708d0ebb3381d257f8ac76eac36","final_merged_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T23:04:31Z","started_at":"2026-10-08T23:04:31Z","completed_at":"2026-10-09T01:20:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T01:31:21.457604774Z","timing_metadata_checked_at":"2026-10-09T01:31:21.457604774Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T01:31:21.461494963Z","region_metadata_checked_at":"2026-10-09T01:31:21.461494963Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7389,"pr_state":"closed","pr_sha":"45fd914d96c1b908f204765007e479aac04b9b32","final_merged_sha":"45fd914d96c1b908f204765007e479aac04b9b32","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T22:02:45Z","started_at":"2026-10-08T22:02:45Z","completed_at":"2026-10-09T00:33:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.630275387Z","timing_metadata_checked_at":"2026-10-09T00:49:25.630275387Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.63268118Z","region_metadata_checked_at":"2026-10-09T00:49:25.63268118Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"30f18516b13b4a29d9c42e4eec22fed43bd7bf5a73e68fce2f799f4abd8b64a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","failed_at":"alert","signature_id":"478c229f000373303094e502fe5867ab11bb46818c55fe4914e070c3ce22b112","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:50Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:50Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"b98cfcdc3f9765687b8b934fa9a541e9873a0baa99572786d4bd824527664a99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:11:31Z\n Ended: 2026-10-08T23:17:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0504704-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317188833153024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T22:02:43Z","started_at":"2026-10-08T22:02:43Z","completed_at":"2026-10-09T00:25:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:39:24.815174246Z","timing_metadata_checked_at":"2026-10-09T00:39:24.815174246Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:39:24.820940528Z","region_metadata_checked_at":"2026-10-09T00:39:24.820940528Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f96f308ee9e09758d9bcffd62e4daa1970649102e66b85faa91702832721e5e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317188833153024","occurred_at":"2026-10-08T22:02:43Z","failed_at":"alert","signature_id":"c904114929467bb32c67ef275024818849dff8d5a227072eff3a386882ff848e","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:04:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3153024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3153024/providers/microsoft.monitor/accounts/hcps-j3153024\", namespace=\"ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-tbmch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:04:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3153024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3153024/providers/microsoft.monitor/accounts/hcps-j3153024\", namespace=\"ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-tbmch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317165428936704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7361,"pr_state":"closed","pr_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","final_merged_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T22:02:37Z","started_at":"2026-10-08T22:02:37Z","completed_at":"2026-10-09T00:35:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.589998106Z","timing_metadata_checked_at":"2026-10-09T00:49:25.589998106Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.592928597Z","region_metadata_checked_at":"2026-10-09T00:49:25.592928597Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317119643914240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T22:02:26Z","started_at":"2026-10-08T22:02:26Z","completed_at":"2026-10-09T00:34:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.652418322Z","timing_metadata_checked_at":"2026-10-09T00:49:25.652418322Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.655197413Z","region_metadata_checked_at":"2026-10-09T00:49:25.655197413Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"77832ce7d4c6526d5e32fa41f2fa6fa6f2e21b193f8edfad98f4ef2bd04d34b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317119643914240","occurred_at":"2026-10-08T22:02:26Z","failed_at":"e2e","signature_id":"eaa2cead8693174d0c57211952caffdb2c67d10b7dc850a37b0ecc67cb24793b","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-7fcmrmvxxv7n/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-7fcmrmvxxv7n/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7364,"pr_state":"closed","pr_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","final_merged_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T22:02:25Z","started_at":"2026-10-08T22:02:25Z","completed_at":"2026-10-09T00:35:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.610275546Z","timing_metadata_checked_at":"2026-10-09T00:49:25.610275546Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.612859438Z","region_metadata_checked_at":"2026-10-09T00:49:25.612859438Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"671b9b4303a83a9249d528ce684b2861186518b9c3215fae7f879616e8b083c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","failed_at":"alert","signature_id":"23b4c1fcca9c095c3310a93e598a980361364ffd7335fed71ecdd85117fddf7e","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"6a280be54f77083aea29bffe587f933b189574d441a83910437e092086143a8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","failed_at":"alert","signature_id":"9dd9aa3c94a6d8b3b046f06122de3e26069f01c693ef0bd9d5a924fe0d0972a0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:31:34Z\n Ended: 2026-10-09T00:04:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4990592-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j4990592-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7180,"pr_state":"closed","pr_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","final_merged_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T22:02:20Z","started_at":"2026-10-08T22:02:20Z","completed_at":"2026-10-09T00:37:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.566952874Z","timing_metadata_checked_at":"2026-10-09T00:49:25.566952874Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.570494763Z","region_metadata_checked_at":"2026-10-09T00:49:25.570494763Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"228e4a13b02a677ae13e9d7f30ae7bfe9c3905a095f7ab8394034e8fe4624495","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","failed_at":"alert","signature_id":"4bef60ee24c495084afb20bb4a919adeb209d9fae5a39e972a9d5a8c119a1d43","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-sst2s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-sst2s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"2d989e194e79c0b37db35e318aa91b4d4b80ae4d5bec235bc4c6d93fbd4049e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:16:25Z\n Ended: 2026-10-08T23:32:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3544576-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"5eac4393c971618396922016b8786257efb05fd5cbb8ae20afa4d0e9fdc775d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","failed_at":"alert","signature_id":"2d0718dd6bc944ecb1f8e90e435ead1ad5ebb1657342b129e97e84fc23cf0c63","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:01:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"cloud-network-config-controller-79b774bd6b-9pkrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6b4e41e1-2282-44a1-ab27-d2b7bffdbb7d\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller-79b774bd6b-9pkrz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:01:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"csi-snapshot-controller-7fcf9f696b-5dd5v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ade41fab-9ae3-4c1c-af6d-d25dc46782e7\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/csi-snapshot-controller-7fcf9f696b-5dd5v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T00:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"csi-snapshot-controller-7fcf9f696b-5dd5v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ade41fab-9ae3-4c1c-af6d-d25dc46782e7\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/csi-snapshot-controller-7fcf9f696b-5dd5v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T00:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"cloud-network-config-controller-79b774bd6b-9pkrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6b4e41e1-2282-44a1-ab27-d2b7bffdbb7d\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller-79b774bd6b-9pkrz (controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"b8fc0743bf90fd2e509f8b80179eee004545d0c72cc218308369e917b11b5a63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","failed_at":"e2e","signature_id":"b9572568edad6e488a7ae0195c7620f5eca412060505066f9072aea06ab756cc","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2c4b80612378\u003e: \n resource group \"ingress-cert-le-68vr9hbvh2st\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-68vr9hbvh2st\" to finish deleting: GET https://rp.j3544576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\",\n \"name\": \"ca06aaf0-429b-4724-9342-8885fc527050\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T23:18:31.808957419Z\",\n \"endTime\": \"2026-10-08T23:42:33.737902461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tat5kosr5birnmo003enu4ag9tvkblp still exists (deletion dispatched at 2026-10-08T23:18:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2c4b80612378\u003e: \n resource group \"ingress-cert-le-68vr9hbvh2st\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-68vr9hbvh2st\" to finish deleting: GET https://rp.j3544576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\",\n \"name\": \"ca06aaf0-429b-4724-9342-8885fc527050\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T23:18:31.808957419Z\",\n \"endTime\": \"2026-10-08T23:42:33.737902461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tat5kosr5birnmo003enu4ag9tvkblp still exists (deletion dispatched at 2026-10-08T23:18:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108316538086887424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"2be9f130d5be3c5ee0a7d837272b65a2bfadc0a0","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T22:00:07Z","started_at":"2026-10-08T22:00:07Z","completed_at":"2026-10-09T00:27:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:39:24.841317464Z","timing_metadata_checked_at":"2026-10-09T00:39:24.841317464Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:39:24.845774349Z","region_metadata_checked_at":"2026-10-09T00:39:24.845774349Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6aea440a1726ce9e4a28e60bb3fbca1480f647d3a1f161f06796b45f1e562769","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108316538086887424","occurred_at":"2026-10-08T22:00:07Z","failed_at":"e2e","signature_id":"7e465cdb4b8db885d2f1d5a2350e3f71d22bf5070b017c7982cfbb0f329e79aa","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x14550d98c9a0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-p98fbdr6lfb6\": PUT https://rp.j6887424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p98fbdr6lfb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x14550d98c9a0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-p98fbdr6lfb6\": PUT https://rp.j6887424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p98fbdr6lfb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108311064448864256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"40bd04949907e4fe11c0550633f35a2867e7a97d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T21:38:22Z","started_at":"2026-10-08T21:38:22Z","completed_at":"2026-10-09T00:04:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:19:15.416198947Z","timing_metadata_checked_at":"2026-10-09T00:19:15.416198947Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:19:15.420202735Z","region_metadata_checked_at":"2026-10-09T00:19:15.420202735Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a30c52ff7d17c3ff70137da7eade84751dbefafdb1ad2da03667cda0c253e56f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108311064448864256","occurred_at":"2026-10-08T21:38:22Z","failed_at":"e2e","signature_id":"87480de3ded9bd25bdf3480fe553bfa947aa5a80228ef448c08c54751c457c7d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1044cd538270\u003e: \n resource group \"rg-hs-pre--l7mzsgckn5bq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-m4xdbw\" in resourcegroup=\"rg-hs-pre--l7mzsgckn5bq\" to finish deleting: GET https://rp.j8864256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"name\": \"81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T22:52:33.320294122Z\",\n \"endTime\": \"2026-10-08T23:16:35.834245337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tasnkqnpkt06eq3lmg9dsk6e9purnb2 still exists (deletion dispatched at 2026-10-08T22:52:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1044cd538270\u003e: \n resource group \"rg-hs-pre--l7mzsgckn5bq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-m4xdbw\" in resourcegroup=\"rg-hs-pre--l7mzsgckn5bq\" to finish deleting: GET https://rp.j8864256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"name\": \"81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T22:52:33.320294122Z\",\n \"endTime\": \"2026-10-08T23:16:35.834245337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tasnkqnpkt06eq3lmg9dsk6e9purnb2 still exists (deletion dispatched at 2026-10-08T22:52:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108293038869581824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7431,"pr_state":"closed","pr_sha":"4324bfa626a4b9b780fef24a9e25c4af822abf4b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T20:26:45Z","started_at":"2026-10-08T20:26:45Z","completed_at":"2026-10-08T22:55:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:07:02.380711038Z","timing_metadata_checked_at":"2026-10-08T23:07:02.380711038Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:07:02.38357343Z","region_metadata_checked_at":"2026-10-08T23:07:02.38357343Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108291662634553344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"1fc9a8ec2d2a2ede4d690afbd9a0636fb7ec01fd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T20:21:17Z","started_at":"2026-10-08T20:21:17Z","completed_at":"2026-10-08T22:48:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:07:02.360509401Z","timing_metadata_checked_at":"2026-10-08T23:07:02.360509401Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:07:02.364161789Z","region_metadata_checked_at":"2026-10-08T23:07:02.364161789Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5ea32972a4c7b7a9b06ae953d36144e2c5555b67951d46e7fce7df566fe4dc65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108291662634553344","occurred_at":"2026-10-08T20:21:17Z","failed_at":"e2e","signature_id":"108516cb0647ba3d2f4c87e8257ac1527a6e7121e308f5f5579382713182c90a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-ttr5gf9j2z4p/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-ttr5gf9j2z4p/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108288696733143040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"open","pr_sha":"6210b7bdb19602dd76919df4a74af1eac0dabffe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T20:09:30Z","started_at":"2026-10-08T20:09:30Z","completed_at":"2026-10-08T20:24:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:39:21.129754415Z","timing_metadata_checked_at":"2026-10-08T20:39:21.129754415Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-08T20:39:21.133914902Z","region_metadata_checked_at":"2026-10-08T21:42:15.696824581Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e663721702d80cbc43acc343a10bd66f864f129c7973ce54bae21b36e0258105","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108288696733143040","occurred_at":"2026-10-08T20:09:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108286667755360256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"closed","pr_sha":"4f5bad547f61d8ae4e3fac3f12a1c2826596c8b8","final_merged_sha":"f2342694ca433fce7c8afb88f010b598afa14625","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T20:01:17Z","started_at":"2026-10-08T20:01:17Z","completed_at":"2026-10-08T22:12:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T22:24:29.408978929Z","timing_metadata_checked_at":"2026-10-08T22:24:29.408978929Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T22:24:29.41279222Z","region_metadata_checked_at":"2026-10-08T22:24:29.41279222Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108281529313529856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7422,"pr_state":"closed","pr_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","final_merged_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T19:41:01Z","started_at":"2026-10-08T19:41:01Z","completed_at":"2026-10-08T22:12:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T22:24:29.541907027Z","timing_metadata_checked_at":"2026-10-08T22:24:29.541907027Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T22:24:29.54466142Z","region_metadata_checked_at":"2026-10-08T22:24:29.54466142Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"c5bf1e12136aa94c4727812ed1c4ec11c47d58ed","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T18:12:42Z","started_at":"2026-10-08T18:12:42Z","completed_at":"2026-10-08T20:39:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:50:10.977991009Z","timing_metadata_checked_at":"2026-10-08T20:50:10.977991009Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:50:10.984901488Z","region_metadata_checked_at":"2026-10-08T20:50:10.984901488Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8cb1acee69a6c117eb2e1e2138539446cea38a6bd485e89b1fdc0cc2ff4780dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","failed_at":"e2e","signature_id":"9bbb9a304b3e216bd163fb308a5bc70d9beeba6ca91eeefd3ac9063123943dc7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36065ae1e700\u003e: \n failed to create NodePool np-2b5cwp: failed waiting for nodepool=\"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\" in resourcegroup=\"rg-candidate-5-1-mdbffxtcx8wr\" to finish creating: GET https://rp.j6247936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"name\": \"e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T19:25:31.299024517Z\",\n \"endTime\": \"2026-10-08T19:54:32.257674658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36065ae1e700\u003e: \n failed to create NodePool np-2b5cwp: failed waiting for nodepool=\"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\" in resourcegroup=\"rg-candidate-5-1-mdbffxtcx8wr\" to finish creating: GET https://rp.j6247936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"name\": \"e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T19:25:31.299024517Z\",\n \"endTime\": \"2026-10-08T19:54:32.257674658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bd60d621c25b0e728935935c9a1472f5bad0421b2df4dc83021e4c5121bc46a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","failed_at":"alert","signature_id":"7e6534085d42dc6118c7a37dc3076a5592621ad332aa0f0dc2d28d01dd263625","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:58:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T19:59:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108258997998981120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7433,"pr_state":"closed","pr_sha":"b01b6c11e479c075d88a9d3cb084529779b2b3dc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T18:11:29Z","started_at":"2026-10-08T18:11:29Z","completed_at":"2026-10-08T19:57:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:17:50.542007288Z","timing_metadata_checked_at":"2026-10-08T20:17:50.542007288Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:17:50.545195878Z","region_metadata_checked_at":"2026-10-08T20:17:50.545195878Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3e735b8cf1a573f0387be29b73e82ab827cccea2c04e78ac25c2789d5aee2019","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108258997998981120","occurred_at":"2026-10-08T18:11:29Z","failed_at":"alert","signature_id":"da0348e89e81da96d29b328c4681e84b359b3ea85a39dbf681fbeba48167e0dd","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-2g4tl\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-2g4tl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-l5mn9\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-l5mn9 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vfshb\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-vfshb has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-qhgdx\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-qhgdx has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-wm888\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-wm888 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-t9gmx\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-t9gmx has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-h7lkm\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-h7lkm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-w5s5h\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-w5s5h has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vwbm7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-vwbm7 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-n644b\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-n644b has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-gh9b9\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-gh9b9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256282895978496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7361,"pr_state":"closed","pr_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","final_merged_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T18:00:39Z","started_at":"2026-10-08T18:00:39Z","completed_at":"2026-10-08T20:36:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:50:11.496513227Z","timing_metadata_checked_at":"2026-10-08T20:50:11.496513227Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:50:11.5051899Z","region_metadata_checked_at":"2026-10-08T20:50:11.5051899Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c00a8b11a1d9ae4f8e353dbbb4b2605cd115e147fc66e3c4e4d08bc5d6877f90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256282895978496","occurred_at":"2026-10-08T18:00:39Z","failed_at":"alert","signature_id":"5a7e6f416bdd814aa6842042551270c54b63893bd3ddd65b8fe96afa706bcb83","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:32Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j5978496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5978496/providers/microsoft.monitor/accounts/services-j5978496\", namespace=\"klusterlet-2tapn8dlbj9p40f8ecef335ah1n44qgl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-snrs6\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tapn8dlbj9p40f8ecef335ah1n44qgl on cluster ci01-j5978496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T18:00:34Z","started_at":"2026-10-08T18:00:34Z","completed_at":"2026-10-08T20:40:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:00:16.627028343Z","timing_metadata_checked_at":"2026-10-08T21:00:16.627028343Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T21:00:16.637362215Z","region_metadata_checked_at":"2026-10-08T21:00:16.637362215Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"a2a67e9f1ec708cdbaf552a536dfe4b2c706d8ee14204ce7cc285f66a148ff91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","failed_at":"alert","signature_id":"4f9f8db4cb3f25b70a751ee41026225f752fc0d2a21726d68e44cf77af9d8f9f","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T20:22:48Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T20:23:48Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"cb52a52792694b0fb6757fb5c62e36843990a36a9af5e40d6f66eb1a7df8abbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T19:18:13Z\n Ended: 2026-10-08T19:24:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8343424-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"open","pr_sha":"91559182869d9c499a23720ed1842a2f04eb10f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T18:00:26Z","started_at":"2026-10-08T18:00:26Z","completed_at":"2026-10-08T21:53:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T22:13:41.757118558Z","timing_metadata_checked_at":"2026-10-08T22:13:41.757118558Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T22:13:41.761544744Z","region_metadata_checked_at":"2026-10-08T22:13:41.761544744Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"08fac5ed2671b9b3698349994c9f27c98c8ec41460aa6e8af3f00718c3cc6ebc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","failed_at":"e2e","signature_id":"e52340d4b382cdd2542ada160fb67461b260ba9f52c641a312b0e9e87e132a0b","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d9f3c92fc0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-gcnbkpsxtjrv\" to finish creating: GET https://rp.j5951872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"name\": \"511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T20:56:10.667754986Z\",\n \"endTime\": \"2026-10-08T21:25:14.560841698Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d9f3c92fc0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-gcnbkpsxtjrv\" to finish creating: GET https://rp.j5951872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"name\": \"511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T20:56:10.667754986Z\",\n \"endTime\": \"2026-10-08T21:25:14.560841698Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c34532dbcbfa9842a91623fc59bdbad202171fd8167fc28efc2b21e98606f30d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","failed_at":"alert","signature_id":"1e60a482d86ac260f86e568bf243b409c0a25d681cfd32fd6d4ce13302592f5a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T21:07:52Z\n Ended: 2026-10-08T21:13:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5951872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tar6kfvsbfkgd4sfo2s5ak9up9mghnd-ephemeral-disk\", pod=\"openshift-apiserver-f7bf46fd6-jsjg9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tar6kfvsbfkgd4sfo2s5ak9up9mghnd-ephemeral-disk/openshift-apiserver-f7bf46fd6-jsjg9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7389,"pr_state":"closed","pr_sha":"45fd914d96c1b908f204765007e479aac04b9b32","final_merged_sha":"45fd914d96c1b908f204765007e479aac04b9b32","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T18:00:25Z","started_at":"2026-10-08T18:00:25Z","completed_at":"2026-10-08T20:34:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:50:12.51755331Z","timing_metadata_checked_at":"2026-10-08T20:50:12.51755331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:50:12.528180278Z","region_metadata_checked_at":"2026-10-08T20:50:12.528180278Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"4b9e81ed3523ebeca07b11b06695666e960cb62c91217f181d51ac040fa861f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T19:16:33Z\n Ended: 2026-10-08T19:22:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6052992-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"b461490a748dc60c2cfb550c9b7e027886593f409b3bd288a14c56aafa581e01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","failed_at":"alert","signature_id":"481b22a2ae9014aee1f61475b4be429b12de22c8d096e16cf3cf96337a687bf6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:58:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6052992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6052992/providers/microsoft.monitor/accounts/hcps-j6052992\", namespace=\"ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vc7b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T20:01:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6052992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6052992/providers/microsoft.monitor/accounts/hcps-j6052992\", namespace=\"ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vc7b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T20:18:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6052992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6052992/providers/microsoft.monitor/accounts/hcps-j6052992\", namespace=\"ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vc7b6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapkvkaamqqutsmtseph2ro2qdkc8vv-d3m0g1e7k9t9s9o/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256210653286400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7180,"pr_state":"closed","pr_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","final_merged_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T18:00:24Z","started_at":"2026-10-08T18:00:24Z","completed_at":"2026-10-08T20:32:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:50:11.173095414Z","timing_metadata_checked_at":"2026-10-08T20:50:11.173095414Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:50:11.181718687Z","region_metadata_checked_at":"2026-10-08T20:50:11.181718687Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"46af1073438afb79a781429d4282dd5fb42ca3beffbb70a96248648179c26d22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256210653286400","occurred_at":"2026-10-08T18:00:24Z","failed_at":"e2e","signature_id":"75c025839b4b77bebd3fbfe7d5be33a670bec55dc16b0fc470c5b7fcd3c0ad20","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108248385097043968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7371,"pr_state":"open","pr_sha":"f3c24f06ee5fec945aea29299574075ed4d059d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T17:29:18Z","started_at":"2026-10-08T17:29:18Z","completed_at":"2026-10-08T20:15:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:28:40.658400484Z","timing_metadata_checked_at":"2026-10-08T20:28:40.658400484Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:28:40.661874974Z","region_metadata_checked_at":"2026-10-08T20:28:40.661874974Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d566b585e816d336bd9fe170323135f137e28acc80cb06ef1d055cd60431a4ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108248385097043968","occurred_at":"2026-10-08T17:29:18Z","failed_at":"alert","signature_id":"293fb0dcb3ed0945c0a68b097fa102ed57356450154c2fa182d4761ee486de09","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:44:43Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7043968-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.30:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7043968/providers/microsoft.monitor/accounts/services-j7043968\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6fdbbdc5f8-bb886\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7043968-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T19:44:43Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7043968-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.30:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7043968/providers/microsoft.monitor/accounts/services-j7043968\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6fdbbdc5f8-bb886\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7043968-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7422,"pr_state":"closed","pr_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","final_merged_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T17:21:25Z","started_at":"2026-10-08T17:21:25Z","completed_at":"2026-10-08T18:50:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T18:53:20.047521716Z","timing_metadata_checked_at":"2026-10-08T18:53:20.047521716Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T18:53:20.051577803Z","region_metadata_checked_at":"2026-10-08T18:53:20.051577803Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"20221574deed9afbe7ced1117bda7ab47f6513c25be25e42a00340198ebc58a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","failed_at":"alert","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"628c7c847930d9f6064d7aa496a1636a1bf627b3b4e086dcb9aaf6903a393d76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"d775ae5c21b8858a5d37c5c400af1be4c12608e32c34cb2972fc8723def0ffb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108235891553079296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7248,"pr_state":"open","pr_sha":"6da1dff5c20d3661c66199e6b5fcaef98fa595ed","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T16:39:40Z","started_at":"2026-10-08T16:39:40Z","completed_at":"2026-10-08T19:24:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:25:47.978525673Z","timing_metadata_checked_at":"2026-10-08T19:25:47.978525673Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:25:47.981388864Z","region_metadata_checked_at":"2026-10-08T19:25:47.981388864Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108232019417763840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7403,"pr_state":"closed","pr_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","final_merged_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T16:24:17Z","started_at":"2026-10-08T16:24:17Z","completed_at":"2026-10-08T19:02:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:03:59.172341235Z","timing_metadata_checked_at":"2026-10-08T19:03:59.172341235Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:03:59.176403422Z","region_metadata_checked_at":"2026-10-08T19:03:59.176403422Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"1fc9a8ec2d2a2ede4d690afbd9a0636fb7ec01fd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T16:13:48Z","started_at":"2026-10-08T16:13:48Z","completed_at":"2026-10-08T19:21:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:25:47.780855807Z","timing_metadata_checked_at":"2026-10-08T19:25:47.780855807Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:25:47.784142996Z","region_metadata_checked_at":"2026-10-08T19:25:47.784142996Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"09e8567983f22af2da4691abcb586ab595759bab537a0327af08aca796b36fa0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","failed_at":"alert","signature_id":"9a5683b6fb4e970df594707347166c3a5464fde6e4de17ed87d6680996438cdc","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T18:47:49Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9147264-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9147264/providers/microsoft.monitor/accounts/services-j9147264\", namespace=\"klusterlet-2taok2013r1ut7fvo0ulo5kc534ntq7s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-fljb2\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2taok2013r1ut7fvo0ulo5kc534ntq7s on cluster ci01-j9147264-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"46059e3c4a95d321df5334f2ef3d2daf1cfa188bf941027aefb1b2a6a1054ab8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","failed_at":"alert","signature_id":"dea7649e1e363ea4afbe251966c45c45a904d878aa58e50d8b09b9fb6f157106","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T18:18:15Z\n Ended: 2026-10-08T18:25:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9147264-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2taok4lfsu899t43cjah6nqtn60rbr6d-r8d4l6w3w4r4h6t\", pod=\"openshift-apiserver-59b5875fdc-r9dd5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2taok4lfsu899t43cjah6nqtn60rbr6d-r8d4l6w3w4r4h6t/openshift-apiserver-59b5875fdc-r9dd5 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"942a790e24ead9bb31e3db07be6c4337f14148b9a7d54451ffd7f7b16e6baa46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T18:03:54Z\n Ended: 2026-10-08T18:09:56Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9147264-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"a7adb70dad227e3de8ae62264421119b8ddfccc9a7e2cd2e8ec15af5c7a44897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","failed_at":"e2e","signature_id":"c584d03b77f63107c8174575f5f10a9efc54b55c500b0c9908081f3f3d257f5a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229184860983296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"closed","pr_sha":"d4e966035e6c45f467a42629c6a7e6844498ff80","final_merged_sha":"f2342694ca433fce7c8afb88f010b598afa14625","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T16:13:01Z","started_at":"2026-10-08T16:13:01Z","completed_at":"2026-10-08T18:26:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T18:31:40.808675455Z","timing_metadata_checked_at":"2026-10-08T18:31:40.808675455Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T18:31:40.811597845Z","region_metadata_checked_at":"2026-10-08T18:31:40.811597845Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7325,"pr_state":"closed","pr_sha":"e290a1070502fcf11020b6c299a9edb0e7f6c9dc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T16:11:50Z","started_at":"2026-10-08T16:11:50Z","completed_at":"2026-10-08T18:35:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T18:42:35.961962181Z","timing_metadata_checked_at":"2026-10-08T18:42:35.961962181Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T18:42:35.965852869Z","region_metadata_checked_at":"2026-10-08T18:42:35.965852869Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"47e42a3b33bd5df2f40022ea9dd9dc61b1d52e6591cd4c01dba6f885706968fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","failed_at":"e2e","signature_id":"a43f99b00f1bbd4a520785c52599fad0e47aab9b8859a5d6eb181cda1291d1d6","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xed922aab58\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xed922aab58\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"d0489730878c0c9e24de2d48724b6d03532ff7c0554a730987d56c5c0e6f1f6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","failed_at":"e2e","signature_id":"7ca525a80ef321179f5efc8d43f4c69d8ec239f2d58e9a339c8fd79d95ff3fb1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1c054b45c180\u003e: \n resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1c054b45c180\u003e: \n resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2ff1c1a68d0b7eb8feafc9a1cf5263e0febf6027fd02f7d1034028223fcef46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","failed_at":"e2e","signature_id":"8db99276b1e37ca2b73c952dfac288a0b7ae6c24b3ed28e03f5e617ef6f4ab9c","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126c34d780d8\u003e: \n resource group \"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-d2kz66\" in resourcegroup=\"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"name\": \"53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:58:56.699498743Z\",\n \"endTime\": \"2026-10-08T18:23:02.391989177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tao6fb6en2d65csg2b2sr0ei58fvhru still exists (deletion dispatched at 2026-10-08T17:58:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126c34d780d8\u003e: \n resource group \"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-d2kz66\" in resourcegroup=\"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"name\": \"53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:58:56.699498743Z\",\n \"endTime\": \"2026-10-08T18:23:02.391989177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tao6fb6en2d65csg2b2sr0ei58fvhru still exists (deletion dispatched at 2026-10-08T17:58:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224421582868480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7403,"pr_state":"closed","pr_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","final_merged_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T15:54:05Z","started_at":"2026-10-08T15:54:05Z","completed_at":"2026-10-08T16:14:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:24:16.069159807Z","timing_metadata_checked_at":"2026-10-08T16:24:16.069159807Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-08T16:24:16.073748192Z","region_metadata_checked_at":"2026-10-08T17:28:27.781447112Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b371fed582cf31afb524798c5c6c7269fbfa45cf734eee68472d6655483dd47f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224421582868480","occurred_at":"2026-10-08T15:54:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224410405048320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"40bd04949907e4fe11c0550633f35a2867e7a97d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T15:54:02Z","started_at":"2026-10-08T15:54:02Z","completed_at":"2026-10-08T16:37:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:54:47.297478026Z","timing_metadata_checked_at":"2026-10-08T16:54:47.297478026Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:54:47.301208414Z","region_metadata_checked_at":"2026-10-08T16:54:47.301208414Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"16788fc9674feb82da9395cda788b440b7094d2ef935b2475c13b71855cc0be9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224410405048320","occurred_at":"2026-10-08T15:54:02Z","failed_at":"provision","signature_id":"664aa8f1aeb2650939331a96787558c2e7340b55e45ef2847dd30061d3213023","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T16:19:05.365Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:30:17.342Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:19:05.375275155Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.114395254Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.462375095Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335389178Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6F2FEEE9F3FAC221\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M5.9123598S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335532727Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"98034B352C012EE7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT4.7536001S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.33558093Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DE129F3B0C485FAF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M38.1712414S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335599002Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B325BEA3E7A07E7A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M56.0129258S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335620623Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D7A0CEADFD719B73\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT3.9567105S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335657835Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7F61B179B809D260\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M0.9591654S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335709239Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101305363180932\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"11m11.960448525s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108222712823418880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7364,"pr_state":"closed","pr_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","final_merged_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T15:47:18Z","started_at":"2026-10-08T15:47:18Z","completed_at":"2026-10-08T16:28:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:45.940318785Z","timing_metadata_checked_at":"2026-10-08T16:44:45.940318785Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:45.943302175Z","region_metadata_checked_at":"2026-10-08T16:44:45.943302175Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a1c2308bb9c5524cba2b30403d161b57ed0d6bda1fd6a69817896b722d1a731a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108222712823418880","occurred_at":"2026-10-08T15:47:18Z","failed_at":"provision","signature_id":"94dd4a2e46776ab0e7c5e9b444cb5e7c4246df9918134350ba21a74752e8098b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T16:14:31.581Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:22:39.946Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:14:31.590606721Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.50353792Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.891298669Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944584337Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"241855BF9A0D266F\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT1M38.1681956S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944653829Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5B81E75ADA721336\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.9210349S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.94468287Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"0A9A6CB955E8A5E9\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT7M50.9531763S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944693871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"127F3C3755EDC782\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2491544S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944712361Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1AF780C01A639DF0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2335315S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944730542Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2DBAB0983C574AF5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT5M54.4662985S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944752483Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101308111130568\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m8.354161621s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108221120938250240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"bb59d8f4b3c2afe12b86c7cfdf401e05c5864b5b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T15:40:58Z","started_at":"2026-10-08T15:40:58Z","completed_at":"2026-10-08T20:09:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:28:40.456738271Z","timing_metadata_checked_at":"2026-10-08T20:28:40.456738271Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:28:40.461361458Z","region_metadata_checked_at":"2026-10-08T20:28:40.461361458Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"782ca7e605287ba807e363354257055e424c7459ec30cd9fa2714e55f6080412","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108221120938250240","occurred_at":"2026-10-08T15:40:58Z","failed_at":"alert","signature_id":"27e681f75202e09c64bf1bc1b442c563ec7f3137ae918747ec39eefd467cec74","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T17:46:24Z\n Ended: 2026-10-08T19:30:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8250240-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8250240/providers/microsoft.monitor/accounts/services-j8250240\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5d9b69f6ff-z8bjk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8250240-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Resolved\n Started: 2026-10-08T17:46:24Z\n Ended: 2026-10-08T19:30:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8250240-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8250240/providers/microsoft.monitor/accounts/services-j8250240\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5d9b69f6ff-z8bjk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8250240-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108220296300335104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7431,"pr_state":"closed","pr_sha":"0730dadf26612807184be4c31839e9fa18fe8c80","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T15:37:42Z","started_at":"2026-10-08T15:37:42Z","completed_at":"2026-10-08T18:00:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T18:10:58.252568741Z","timing_metadata_checked_at":"2026-10-08T18:10:58.252568741Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T18:10:58.263056909Z","region_metadata_checked_at":"2026-10-08T18:10:58.263056909Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f863980802aeb4a18745138d4fba6739f5562f13e2fd74ece1500f4dceef5859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108220296300335104","occurred_at":"2026-10-08T15:37:42Z","failed_at":"e2e","signature_id":"d5485ecbaf29eb6153d7283048b59ef21ec89644b21cb87a56aa54d40e6a3fa5","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e49d4d5a340\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj\" to finish creating: GET https://rp.j0335104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\",\n \"name\": \"b0463f50-61f7-4515-9352-022d3d794646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:41:51.052187772Z\",\n \"endTime\": \"2026-10-08T17:10:57.967451326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e49d4d5a340\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj\" to finish creating: GET https://rp.j0335104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\",\n \"name\": \"b0463f50-61f7-4515-9352-022d3d794646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:41:51.052187772Z\",\n \"endTime\": \"2026-10-08T17:10:57.967451326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108217820029390848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7383,"pr_state":"open","pr_sha":"1784d185cf2ec64c02025b1319a28522dbc857bb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T15:27:51Z","started_at":"2026-10-08T15:27:51Z","completed_at":"2026-10-08T17:45:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:49:49.972411423Z","timing_metadata_checked_at":"2026-10-08T17:49:49.972411423Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:49:50.012026597Z","region_metadata_checked_at":"2026-10-08T17:49:50.012026597Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d2365e3db1ad7e0525da570766072baf15fd075707f2ef532557a4b2b62706c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108217820029390848","occurred_at":"2026-10-08T15:27:51Z","failed_at":"e2e","signature_id":"a9e7d98d59a4178bf41e7b5661530ffe552aba4c92d51d87d0a3a1f5a9752100","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108217409373474816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7430,"pr_state":"open","pr_sha":"dbfe4a6556de92ac459983ddfdc1546ea87e7b64","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T15:26:13Z","started_at":"2026-10-08T15:26:13Z","completed_at":"2026-10-08T19:44:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:57:00.246566057Z","timing_metadata_checked_at":"2026-10-08T19:57:00.246566057Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:57:00.250891744Z","region_metadata_checked_at":"2026-10-08T19:57:00.250891744Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108214686754279424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"open","pr_sha":"6210b7bdb19602dd76919df4a74af1eac0dabffe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T15:15:21Z","started_at":"2026-10-08T15:15:21Z","completed_at":"2026-10-08T19:57:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:17:50.411205Z","timing_metadata_checked_at":"2026-10-08T20:17:50.411205Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:17:50.415406386Z","region_metadata_checked_at":"2026-10-08T20:17:50.415406386Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"02b15d157942482025c6f0deb994194d03e47ae84e7649b9a06a3ecca68e4071","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108214686754279424","occurred_at":"2026-10-08T15:15:21Z","failed_at":"e2e","signature_id":"80c9f90b48cd6de845d1ea9fcfc0ec8a6a97df54de2e0676d98bab3ffdbc932e","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2e1efc3180c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\" to finish deleting: GET https://rp.j4279424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"name\": \"7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:07.770334262Z\",\n \"endTime\": \"2026-10-08T16:41:10.588488129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tan1stnb9qf5nfuks8bh5mvhk8g2eul still exists (deletion dispatched at 2026-10-08T16:17:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2e1efc3180c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\" to finish deleting: GET https://rp.j4279424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"name\": \"7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:07.770334262Z\",\n \"endTime\": \"2026-10-08T16:41:10.588488129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tan1stnb9qf5nfuks8bh5mvhk8g2eul still exists (deletion dispatched at 2026-10-08T16:17:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209328392507392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7429,"pr_state":"closed","pr_sha":"1386ec573a6f0619cddb7f54ea1f19284301ed10","final_merged_sha":"1386ec573a6f0619cddb7f54ea1f19284301ed10","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T14:54:07Z","started_at":"2026-10-08T14:54:07Z","completed_at":"2026-10-08T17:32:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:39:06.868096289Z","timing_metadata_checked_at":"2026-10-08T17:39:06.868096289Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:39:06.87101278Z","region_metadata_checked_at":"2026-10-08T17:39:06.87101278Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7361,"pr_state":"closed","pr_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","final_merged_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T14:53:44Z","started_at":"2026-10-08T14:53:44Z","completed_at":"2026-10-08T17:38:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:39:06.092897532Z","timing_metadata_checked_at":"2026-10-08T17:39:06.092897532Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:39:06.095598024Z","region_metadata_checked_at":"2026-10-08T17:39:06.095598024Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6952d810d7242e2d381f88e5827868cd6cea0c7b81e03caa063f1a677cda7673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","failed_at":"e2e","signature_id":"0e4b25ccea79ae9ce774b1501e6333b6a44367fc8f0e789e5ce189b4d807484c","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3adaf74760a8\u003e: \n resource group \"fips-enabled-khqp8vkktpzt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-khqp8vkktpzt\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"name\": \"407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:13:27.229044406Z\",\n \"endTime\": \"2026-10-08T16:37:32.626253468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamus8q8drvpor1j4v73qnjn2pbcpag still exists (deletion dispatched at 2026-10-08T16:13:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3adaf74760a8\u003e: \n resource group \"fips-enabled-khqp8vkktpzt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-khqp8vkktpzt\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"name\": \"407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:13:27.229044406Z\",\n \"endTime\": \"2026-10-08T16:37:32.626253468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamus8q8drvpor1j4v73qnjn2pbcpag still exists (deletion dispatched at 2026-10-08T16:13:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7408e94f8b50bfaad483ea467245e91d648f356125bf1fda38af7fcfbc535e0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","failed_at":"e2e","signature_id":"9f5e90ecbae9c619ce85e90de1ac44b213ae43628780d197bc24085fef9e391f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-l22klm\" for cluster \"cluster-candidate-4-21-l22klm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x320d00821e20\u003e: \n failed to create NodePool np-l22klm: failed waiting for nodepool=\"np-l22klm\" for cluster \"cluster-candidate-4-21-l22klm\" in resourcegroup=\"rg-candidate-4-21-hqlkfl7j7kk5\" to finish creating: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\",\n \"name\": \"e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:12:49.167614937Z\",\n \"endTime\": \"2026-10-08T16:41:52.222456636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn to exist\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn to exist\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-l22klm\" for cluster \"cluster-candidate-4-21-l22klm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x320d00821e20\u003e: \n failed to create NodePool np-l22klm: failed waiting for nodepool=\"np-l22klm\" for cluster \"cluster-candidate-4-21-l22klm\" in resourcegroup=\"rg-candidate-4-21-hqlkfl7j7kk5\" to finish creating: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\",\n \"name\": \"e7ada65d-8a80-4fa7-a9ff-9aa084567ce4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:12:49.167614937Z\",\n \"endTime\": \"2026-10-08T16:41:52.222456636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn to exist\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-sgrrn to exist\\nMachine c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hqlkfl7j7kk5-candidate-l22klm--managed/providers/Microsoft.Compute/virtualMachines/c4a9q0q2m9f5u5s-np-l22klm-wxdn5-txk4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"936aaae597d03bad0f28a93e2914f92b490cba94522f0a89fc26c48910108575","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","failed_at":"e2e","signature_id":"7cbece22670a9f172523e3b354735cd562db62f7e42f6e03ee9317ebdd207c57","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1aa3a56380a8\u003e: \n resource group \"private-kas-mmbkx8tpnrlv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-mmbkx8tpnrlv\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"name\": \"bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:22:19.235799645Z\",\n \"endTime\": \"2026-10-08T16:46:22.547532555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamuoo5fqaudkk7q5op01b86l5utmqh still exists (deletion dispatched at 2026-10-08T16:22:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1aa3a56380a8\u003e: \n resource group \"private-kas-mmbkx8tpnrlv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-mmbkx8tpnrlv\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"name\": \"bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:22:19.235799645Z\",\n \"endTime\": \"2026-10-08T16:46:22.547532555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamuoo5fqaudkk7q5op01b86l5utmqh still exists (deletion dispatched at 2026-10-08T16:22:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"closed","pr_sha":"0d147555918839b4a556ae5981b12356ac4cf337","final_merged_sha":"0d147555918839b4a556ae5981b12356ac4cf337","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T14:46:24Z","started_at":"2026-10-08T14:46:24Z","completed_at":"2026-10-08T17:37:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:39:05.72547239Z","timing_metadata_checked_at":"2026-10-08T17:39:05.72547239Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:39:05.729206478Z","region_metadata_checked_at":"2026-10-08T17:39:05.729206478Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4cb6440528d5d652cc2f83d342171ab97b5644b3482f9ab1ddf99de5052b5064","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","failed_at":"e2e","signature_id":"7a3a06bb18ed74825abe049cd09dc707ee7a940daddfb1a541b6df012fba70c9","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-l8tn4qdvlqv4/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-l8tn4qdvlqv4/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"db04843fba23bbd2a3be7463886b3fba364c31f0500d241540dd227be27c993b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","failed_at":"e2e","signature_id":"13da5141ccb9f5269ca39d7846b3675b49c796a19d5d81feddf02f4d488d863f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x44478a7c150\u003e: \n resource group \"arm64-vm-cluster-ldnw4gs8dwrw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldnw4gs8dwrw\" to finish deleting: GET https://rp.j7078784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"name\": \"fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:57.9406496Z\",\n \"endTime\": \"2026-10-08T16:41:57.233664583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x44478a7c150\u003e: \n resource group \"arm64-vm-cluster-ldnw4gs8dwrw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldnw4gs8dwrw\" to finish deleting: GET https://rp.j7078784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"name\": \"fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:57.9406496Z\",\n \"endTime\": \"2026-10-08T16:41:57.233664583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108206773386088448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"ca4a1701b4d44faba4f03681b85cc336075f5575","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T14:43:55Z","started_at":"2026-10-08T14:43:55Z","completed_at":"2026-10-08T17:13:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:17:49.90309333Z","timing_metadata_checked_at":"2026-10-08T17:17:49.90309333Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:17:49.90626032Z","region_metadata_checked_at":"2026-10-08T17:17:49.90626032Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108206578845880320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7428,"pr_state":"open","pr_sha":"9780e6cb0ea6bdff94f04c84e9bffaab00d80812","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T14:43:11Z","started_at":"2026-10-08T14:43:11Z","completed_at":"2026-10-08T17:32:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T17:39:06.453702995Z","timing_metadata_checked_at":"2026-10-08T17:39:06.453702995Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T17:39:06.456519386Z","region_metadata_checked_at":"2026-10-08T17:39:06.456519386Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108202402925514752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T14:26:35Z","started_at":"2026-10-08T14:26:35Z","completed_at":"2026-10-08T16:45:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:54:47.169304149Z","timing_metadata_checked_at":"2026-10-08T16:54:47.169304149Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:54:47.173220436Z","region_metadata_checked_at":"2026-10-08T16:54:47.173220436Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T14:14:35Z","started_at":"2026-10-08T14:14:35Z","completed_at":"2026-10-08T15:16:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T15:32:14.942146516Z","timing_metadata_checked_at":"2026-10-08T15:32:14.942146516Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T15:32:14.945980004Z","region_metadata_checked_at":"2026-10-08T15:32:14.945980004Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"3780761d115db1e489be5d302399e2cfa89335eb381cec34d707693ad15d6792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","failed_at":"provision","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T14:39:49.369Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-08T14:39:51.193Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-08T14:39:53.050Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1446656-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645\ntime=2026-10-08T15:09:49.053Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}},{"row_id":"457eb5bd70716e11449fdf0dca48cf6047adc6e95612f1a54b845d8a6aa2effd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","failed_at":"alert","signature_id":"22ea0c3803bb98326df9cbf23de00a80472321a99d7d574f14fa44cb6663b005","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T15:04:42Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1446656-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T15:05:40Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1446656-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7422,"pr_state":"closed","pr_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","final_merged_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T14:06:48Z","started_at":"2026-10-08T14:06:48Z","completed_at":"2026-10-08T16:30:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:46.220529602Z","timing_metadata_checked_at":"2026-10-08T16:44:46.220529602Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:46.22376779Z","region_metadata_checked_at":"2026-10-08T16:44:46.22376779Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"aab1ceb06656e9d2f0b989ba43d427fe7887839a84b9d8835000e18a1fc65ee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","failed_at":"alert","signature_id":"b50a31093336d8331ae783c4d9e7666f297d09f74d347cd9133e1c15e42f6f1b","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T16:08:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5460992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.17:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5460992/providers/microsoft.monitor/accounts/hcps-j5460992\", namespace=\"ocm-arohcpci01-2tam3ufd6i9i7hbiu868p9lhfmmiku8i-v4d4z2a2t7q6o5b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jxjxn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tam3ufd6i9i7hbiu868p9lhfmmiku8i-v4d4z2a2t7q6o5b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T16:08:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5460992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.17:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5460992/providers/microsoft.monitor/accounts/hcps-j5460992\", namespace=\"ocm-arohcpci01-2tam3ufd6i9i7hbiu868p9lhfmmiku8i-v4d4z2a2t7q6o5b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jxjxn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tam3ufd6i9i7hbiu868p9lhfmmiku8i-v4d4z2a2t7q6o5b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"d7451ed3d52f1bf62e7f7d576d96404e285fe22175f0a6383a570f3e11018816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","failed_at":"e2e","signature_id":"537d2f8688d12100b662592e26040e8091b73d73dada1ea5844df99344a5080b","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x255cb079f740\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp.j5460992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"name\": \"5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:30:05.316469426Z\",\n \"endTime\": \"2026-10-08T15:59:10.872777291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x255cb079f740\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp.j5460992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"name\": \"5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:30:05.316469426Z\",\n \"endTime\": \"2026-10-08T15:59:10.872777291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108193756921466880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7426,"pr_state":"closed","pr_sha":"5e720bea5c8fa7ed9206de67c3687a763b94a768","final_merged_sha":"5e720bea5c8fa7ed9206de67c3687a763b94a768","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T13:52:07Z","started_at":"2026-10-08T13:52:07Z","completed_at":"2026-10-08T16:34:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:46.339741783Z","timing_metadata_checked_at":"2026-10-08T16:44:46.339741783Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:46.342797672Z","region_metadata_checked_at":"2026-10-08T16:44:46.342797672Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7427,"pr_state":"open","pr_sha":"269733960011d9292bfded60a6aca75da02c364a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T13:42:51Z","started_at":"2026-10-08T13:42:51Z","completed_at":"2026-10-08T15:10:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T15:22:12.213833204Z","timing_metadata_checked_at":"2026-10-08T15:22:12.213833204Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T15:22:12.216637894Z","region_metadata_checked_at":"2026-10-08T15:22:12.216637894Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"130f9a00e48c16a16f9dda0c610ab250888be318edefff3acfeaf168ca3bea50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","failed_at":"alert","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T14:57:45Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4452224-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"b2ae950a8fabbfcc6c5199da148d6dce2ed55283cad222b0e0b229e586d0fbde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","failed_at":"provision","signature_id":"454655692b823febcffc54365e504c69961bb8ad6b414fb6c4922c06be0ce0d9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T14:32:31.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T15:02:30.832Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8d3193ed0a7e2c3492a5c226c9a0bdd4693921b8e35909048c2ef7aed3b7502d","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j4452224-mgmt-1\",\"clusterName\":\"ci01-j445222","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["8d3193ed0a7e2c3492a5c226c9a0bdd4693921b8e35909048c2ef7aed3b7502d","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108189567159898112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"4e80ed5e9152470ec014bd07ba99dcca1104349d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T13:35:35Z","started_at":"2026-10-08T13:35:35Z","completed_at":"2026-10-08T16:30:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:45.828646777Z","timing_metadata_checked_at":"2026-10-08T16:44:45.828646777Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:45.832606163Z","region_metadata_checked_at":"2026-10-08T16:44:45.832606163Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"05ffdf3cc934aaf902d3ffeac8cbe51810266c34a0c60cf1b6296c637380b346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108189567159898112","occurred_at":"2026-10-08T13:35:35Z","failed_at":"e2e","signature_id":"7f9cdf151f9d6c520b9b53bcf22e09916fb547d1ef82e84d3970472cca185ca2","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: 503\nto be \u003c\n \u003cint\u003e: 400\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: 503\nto be \u003c\n \u003cint\u003e: 400","failure_pattern_match":{"status":"clustered","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108182743610298368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7403,"pr_state":"closed","pr_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","final_merged_sha":"2d7e2c4a5f25efaac9221c4a8c901ec5c91a3521","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T13:08:28Z","started_at":"2026-10-08T13:08:28Z","completed_at":"2026-10-08T15:36:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T15:53:01.982700238Z","timing_metadata_checked_at":"2026-10-08T15:53:01.982700238Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T15:53:01.98821622Z","region_metadata_checked_at":"2026-10-08T15:53:01.98821622Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dc7d7d3ae78a8ce5344f90a5ff652aa0de5d5ded151634a4d2f2ac45d3e87cfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108182743610298368","occurred_at":"2026-10-08T13:08:28Z","failed_at":"e2e","signature_id":"2ab2afb758add7ee8595134f90992b6ec46e194592da1dee4e4a8da09cd82ffb","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c27e024540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-6zjcv2t4zcfg\" to finish creating: GET https://rp.j0298368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"name\": \"7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:08:53.200082246Z\",\n \"endTime\": \"2026-10-08T14:38:00.180447244Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c27e024540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-6zjcv2t4zcfg\" to finish creating: GET https://rp.j0298368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"name\": \"7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:08:53.200082246Z\",\n \"endTime\": \"2026-10-08T14:38:00.180447244Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181921119866880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7422,"pr_state":"closed","pr_sha":"0af4bb9afa93734e11e1f582c3c9c11225f0066a","final_merged_sha":"e781509f8c1aa5d21e8b4b325a5aff300ef40c3b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T13:05:12Z","started_at":"2026-10-08T13:05:12Z","completed_at":"2026-10-08T13:23:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T14:18:59.087631134Z","timing_metadata_checked_at":"2026-10-08T14:18:59.087631134Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-08T14:18:59.091477221Z","region_metadata_checked_at":"2026-10-08T15:22:12.065344699Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"36a9a486190e45f0d67bfd991d34bca5760ad39fdf0c08cca4dd33acfc9af03e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181921119866880","occurred_at":"2026-10-08T13:05:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181138659872768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7288,"pr_state":"closed","pr_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","final_merged_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T13:02:01Z","started_at":"2026-10-08T13:02:01Z","completed_at":"2026-10-08T15:30:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T15:42:21.4308445Z","timing_metadata_checked_at":"2026-10-08T15:42:21.4308445Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T15:42:21.434810087Z","region_metadata_checked_at":"2026-10-08T15:42:21.434810087Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"open","pr_sha":"91559182869d9c499a23720ed1842a2f04eb10f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T13:01:55Z","started_at":"2026-10-08T13:01:55Z","completed_at":"2026-10-08T15:43:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:03:41.290956336Z","timing_metadata_checked_at":"2026-10-08T16:03:41.290956336Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:03:41.295409622Z","region_metadata_checked_at":"2026-10-08T16:03:41.295409622Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1cdf0d40734066366246bdc4d583ea6e922bdedc352d0fb2a39db142df64d979","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","failed_at":"e2e","signature_id":"5b0ce0129345f929b9f05e55a452f600982aefa185554902a0b82bcf5372be48","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e5bf9e749f8737ec65673ce5aaf89bc324878bae903ebb6ec2b0453ffece29b8","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33"}},{"row_id":"d8abc3bf9cbb99c930d3e6a53547a1c08137f45c78fb09f6b427e08a62fbe22c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","failed_at":"alert","signature_id":"71513c99645f5a7b46d22a8d1780149946d3f4d432f0cff96d7e41203c90b083","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T15:27:53Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7737472-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7737472/providers/microsoft.monitor/accounts/services-j7737472\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-67bdf9f749-q2zbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7737472-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T15:27:53Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7737472-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7737472/providers/microsoft.monitor/accounts/services-j7737472\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-67bdf9f749-q2zbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7737472-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","e5bf9e749f8737ec65673ce5aaf89bc324878bae903ebb6ec2b0453ffece29b8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108178486299463680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"4c445b18d28849c47b82f686efac4b83bfd87b43","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T12:51:33Z","started_at":"2026-10-08T12:51:33Z","completed_at":"2026-10-08T15:08:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T15:22:11.979405786Z","timing_metadata_checked_at":"2026-10-08T15:22:11.979405786Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T15:22:11.983281773Z","region_metadata_checked_at":"2026-10-08T15:22:11.983281773Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2188d18ea930ad467f2baf6d230ee642775fdfdb3e71a91a843792d94f96ff60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108178486299463680","occurred_at":"2026-10-08T12:51:33Z","failed_at":"e2e","signature_id":"703ff58f90b61266d47daa64e91c49cac059b2fddfc664d613b54ef26eabfb02","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x1657ab71ce40\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rkxd84qqqgg8\": PUT https://rp.j9463680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rkxd84qqqgg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x1657ab71ce40\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rkxd84qqqgg8\": PUT https://rp.j9463680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rkxd84qqqgg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108177887361241088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7426,"pr_state":"closed","pr_sha":"5e720bea5c8fa7ed9206de67c3687a763b94a768","final_merged_sha":"5e720bea5c8fa7ed9206de67c3687a763b94a768","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T12:49:11Z","started_at":"2026-10-08T12:49:11Z","completed_at":"2026-10-08T13:29:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:47:35.348236924Z","timing_metadata_checked_at":"2026-10-08T13:47:35.348236924Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-08T13:47:35.351937912Z","region_metadata_checked_at":"2026-10-08T14:51:02.400734959Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7c6ba589b171336f856df8a11a885048d473707859b879469a63828fe9f8a72e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108177887361241088","occurred_at":"2026-10-08T12:49:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7226,"pr_state":"closed","pr_sha":"04b252b2043ff522f94bc3bcd6c141ff457b7716","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T12:16:52Z","started_at":"2026-10-08T12:16:52Z","completed_at":"2026-10-08T12:34:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:55:20.455369049Z","timing_metadata_checked_at":"2026-10-08T12:55:20.455369049Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:55:20.459327536Z","region_metadata_checked_at":"2026-10-08T12:55:20.459327536Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"019842da7d9ebf5e505a9968eb73f5b3689a0f6f769fa68084f89cd2af619e1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j7950976' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"0ab2e754938aac424eff0fca25793fac4cff6ac08c28085cdacf5688ea613cca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j7950976' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"8f367780429a88b4f2977293add5905f5b54bdc318412684c8055628651652fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","failed_at":"alert","signature_id":"2859658063bed829d162181cb3d76e39201e6bf4d59aca2d6c51092fd846a233","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j7950976' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165708520099840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"open","pr_sha":"91559182869d9c499a23720ed1842a2f04eb10f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T12:00:41Z","started_at":"2026-10-08T12:00:41Z","completed_at":"2026-10-08T12:38:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:55:20.906568174Z","timing_metadata_checked_at":"2026-10-08T12:55:20.906568174Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-08T12:55:20.910130162Z","region_metadata_checked_at":"2026-10-08T13:57:41.404175789Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"74babca19561af1b7d9dfa42458a3e1ef81533880709c4ede33d54a8ed097367","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165708520099840","occurred_at":"2026-10-08T12:00:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7389,"pr_state":"closed","pr_sha":"45fd914d96c1b908f204765007e479aac04b9b32","final_merged_sha":"45fd914d96c1b908f204765007e479aac04b9b32","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T12:00:36Z","started_at":"2026-10-08T12:00:36Z","completed_at":"2026-10-08T14:25:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T14:40:13.319440111Z","timing_metadata_checked_at":"2026-10-08T14:40:13.319440111Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T14:40:13.333394965Z","region_metadata_checked_at":"2026-10-08T14:40:13.333394965Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"05b3cd830a3be049689b097e61c96fe9a87d40d0bc43c5bcbf9ab42c309cadc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","failed_at":"alert","signature_id":"13fd8fddd2d9f41c17b15476dbb1a6d18d3c125423d8872d5b4d0619fa486992","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T13:48:14Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9653888-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.155:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9653888/providers/microsoft.monitor/accounts/services-j9653888\", namespace=\"klusterlet-2tak8n7sjj3mgsij1id8amijusdtem6a\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-87ql4\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tak8n7sjj3mgsij1id8amijusdtem6a on cluster ci01-j9653888-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"82c0db9597fbd188b3d019f5fa6c5835351254d3f4543de3272e8e54be828b9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T13:09:26Z\n Ended: 2026-10-08T13:15:29Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9653888-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"97dabc408c5c896a6ff29d7a6d1c54d5f371f234623fb0b2f684ca8535a761c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","failed_at":"e2e","signature_id":"6089f291ebd2776303b1bbe50c4d8aa3d633aed985bf844efdbe7aacda4c2397","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-54jpd8\" for cluster \"cluster-candidate-5-1-54jpd8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18bafeade620\u003e: \n failed to create NodePool np-54jpd8: failed waiting for nodepool=\"np-54jpd8\" for cluster \"cluster-candidate-5-1-54jpd8\" in resourcegroup=\"rg-candidate-5-1-ch7fc46qsw9p\" to finish creating: GET https://rp.j9653888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b250e4db-d610-4d88-ac6c-923c943777b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b250e4db-d610-4d88-ac6c-923c943777b1\",\n \"name\": \"b250e4db-d610-4d88-ac6c-923c943777b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T13:07:33.724991852Z\",\n \"endTime\": \"2026-10-08T13:36:35.125261835Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll to exist\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll to exist\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-54jpd8\" for cluster \"cluster-candidate-5-1-54jpd8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18bafeade620\u003e: \n failed to create NodePool np-54jpd8: failed waiting for nodepool=\"np-54jpd8\" for cluster \"cluster-candidate-5-1-54jpd8\" in resourcegroup=\"rg-candidate-5-1-ch7fc46qsw9p\" to finish creating: GET https://rp.j9653888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b250e4db-d610-4d88-ac6c-923c943777b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b250e4db-d610-4d88-ac6c-923c943777b1\",\n \"name\": \"b250e4db-d610-4d88-ac6c-923c943777b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T13:07:33.724991852Z\",\n \"endTime\": \"2026-10-08T13:36:35.125261835Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll to exist\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-cshll to exist\\nMachine a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-ch7fc46qsw9p-candidate-54jpd8--managed/providers/Microsoft.Compute/virtualMachines/a9w9k0t2v4a5r8d-np-54jpd8-ctzwh-nqwtj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3d7f02db20a193fcdc2152484969dfeb720fb771c6684c735871933ba7b1ae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","failed_at":"e2e","signature_id":"65b7540a6ca0b846f6cc9d88ae9a4e6476ca721e437f95cf3292bc42c3aa02d4","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108163022403932160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7401,"pr_state":"open","pr_sha":"86e55827116f723bc2526ef2eb672efee0f66c22","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T11:50:06Z","started_at":"2026-10-08T11:50:06Z","completed_at":"2026-10-08T14:26:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T14:40:13.732623849Z","timing_metadata_checked_at":"2026-10-08T14:40:13.732623849Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T14:40:13.739482326Z","region_metadata_checked_at":"2026-10-08T14:40:13.739482326Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f55671e3cf4ff81085c47073c14cc2e5a0aed5b3b56d869385882ba7dc19c91f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108163022403932160","occurred_at":"2026-10-08T11:50:06Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T13:21:47Z\n Ended: 2026-10-08T13:27:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3932160-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7383,"pr_state":"open","pr_sha":"d300ed5d2b2370c25ac0094ff82523eb4a0ffa46","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T11:12:24Z","started_at":"2026-10-08T11:12:24Z","completed_at":"2026-10-08T13:39:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:57:41.247936067Z","timing_metadata_checked_at":"2026-10-08T13:57:41.247936067Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T13:57:41.252116154Z","region_metadata_checked_at":"2026-10-08T13:57:41.252116154Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0441182a46b86a3dc9054a6c1b7be1fd6540962d01e3c9fba969c3daf3261cbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","failed_at":"alert","signature_id":"2bb5f552ea27ec9022c5035d204ea8f422b813234e438a587a8af153658a6644","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T13:12:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2977792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2977792/providers/microsoft.monitor/accounts/hcps-j2977792\", namespace=\"ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gngm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T13:12:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2977792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2977792/providers/microsoft.monitor/accounts/hcps-j2977792\", namespace=\"ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gngm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T13:12:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2977792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2977792/providers/microsoft.monitor/accounts/hcps-j2977792\", namespace=\"ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gngm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-08T13:12:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2977792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2977792/providers/microsoft.monitor/accounts/hcps-j2977792\", namespace=\"ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gngm8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tajia53q41l8grpaq7k8eu3qvn81ojn-v20260901/cluster-api has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6accb61d8f24facb592585ec7fb21aa7cd42fdbc03ee11356e23038b5c8bf679","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T12:21:55Z\n Ended: 2026-10-08T12:27:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2977792-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"bfa9a7315a106f99ae7af45558431cb3d3f225fd93cd9f00dfbd4b55b98e710a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","failed_at":"e2e","signature_id":"2fd4ac18224462c0996aff2443f5feed4b4cd3d6b83054edc878630f518b2ab6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7364,"pr_state":"closed","pr_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","final_merged_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T11:09:46Z","started_at":"2026-10-08T11:09:46Z","completed_at":"2026-10-08T13:36:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:47:35.129034947Z","timing_metadata_checked_at":"2026-10-08T13:47:35.129034947Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T13:47:35.135480726Z","region_metadata_checked_at":"2026-10-08T13:47:35.135480726Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"35bffbb8c4670fc0ef3a5895530421183d26c14c42a90abe68cf64473b649a68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","failed_at":"alert","signature_id":"e82e51d2e215239d7b4cd935e809cb5992c44a5d389b3abdcf1a7fc37def35f8","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T13:21:27Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j2954368-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.126:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2954368/providers/microsoft.monitor/accounts/services-j2954368\", namespace=\"klusterlet-2tajh66uh8t9dscan4deki4hln9el07b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-kbbmz\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tajh66uh8t9dscan4deki4hln9el07b on cluster ci01-j2954368-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"fd8e45a12bf2c3da1a0f46ac5d259f63dd51e65a38ae3a9f895b96184c1e47fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T12:19:34Z\n Ended: 2026-10-08T12:30:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2954368-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7423/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152631049326592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7423,"pr_state":"closed","pr_sha":"472378dc592554d6703f9b2214d0166282bdf661","final_merged_sha":"472378dc592554d6703f9b2214d0166282bdf661","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T11:08:34Z","started_at":"2026-10-08T11:08:34Z","completed_at":"2026-10-08T13:40:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:57:41.754890515Z","timing_metadata_checked_at":"2026-10-08T13:57:41.754890515Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T13:57:41.757577307Z","region_metadata_checked_at":"2026-10-08T13:57:41.757577307Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"c5bf1e12136aa94c4727812ed1c4ec11c47d58ed","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T11:08:06Z","started_at":"2026-10-08T11:08:06Z","completed_at":"2026-10-08T13:51:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T14:08:33.127593055Z","timing_metadata_checked_at":"2026-10-08T14:08:33.127593055Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T14:08:33.131785142Z","region_metadata_checked_at":"2026-10-08T14:08:33.131785142Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1c4f94fe4336ffa41656aac6ff870b6ed80d72b884567e133d720dc4c4f5ca40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","failed_at":"alert","signature_id":"b3859c09791b6a1dee073ca99210a9b01905c300d49723024d404dda969c5d31","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T13:31:01Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j3688704-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.89:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3688704/providers/microsoft.monitor/accounts/services-j3688704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-77896677c6-bt8h2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j3688704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T13:31:01Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j3688704-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.89:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3688704/providers/microsoft.monitor/accounts/services-j3688704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-77896677c6-bt8h2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j3688704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"98f51a12fd3f4efe647aee858b38b95be6611caf0e303c4d5c8d52b90e2d31a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","failed_at":"e2e","signature_id":"1cc5d051835a1c8eb905dcafe82dc37af6a69b3130687b92e4a9ee1058c03a32","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:160]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d2a1467ee00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t4pdz4zt795h\" to finish creating: GET https://rp.j3688704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\",\n \"name\": \"b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:56:17.97671191Z\",\n \"endTime\": \"2026-10-08T13:25:25.246131122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t4pdz4zt795h--managed/providers/Microsoft.Compute/virtualMachines/q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t4pdz4zt795h--managed/providers/Microsoft.Compute/virtualMachines/q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:160]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d2a1467ee00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t4pdz4zt795h\" to finish creating: GET https://rp.j3688704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\",\n \"name\": \"b5864c8a-d5bc-47ca-a843-6cbfb4efb9b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:56:17.97671191Z\",\n \"endTime\": \"2026-10-08T13:25:25.246131122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t4pdz4zt795h--managed/providers/Microsoft.Compute/virtualMachines/q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t4pdz4zt795h--managed/providers/Microsoft.Compute/virtualMachines/q0i4e6e7j3u1g3m-np-1-mkz4r-gn4c2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ab76553aa97e14d4c551a15713579bfa3b9f51bc6b7cdf5c7d9612374ca1c3c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T12:26:40Z\n Ended: 2026-10-08T12:42:41Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3688704-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108139111427608576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7378,"pr_state":"closed","pr_sha":"7bed7b418cbe61b464069212e28bd0e360d80b05","final_merged_sha":"7bed7b418cbe61b464069212e28bd0e360d80b05","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T10:15:05Z","started_at":"2026-10-08T10:15:05Z","completed_at":"2026-10-08T12:47:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:05:56.191789601Z","timing_metadata_checked_at":"2026-10-08T13:05:56.191789601Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T13:05:56.196018688Z","region_metadata_checked_at":"2026-10-08T13:05:56.196018688Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108138688436244480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7407,"pr_state":"closed","pr_sha":"9a791da2eec3588b8692833ae2aec0e30ffff9c5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T10:13:25Z","started_at":"2026-10-08T10:13:25Z","completed_at":"2026-10-08T12:49:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T13:05:56.626160454Z","timing_metadata_checked_at":"2026-10-08T13:05:56.626160454Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T13:05:56.628941145Z","region_metadata_checked_at":"2026-10-08T13:05:56.628941145Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7288,"pr_state":"closed","pr_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","final_merged_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T09:41:34Z","started_at":"2026-10-08T09:41:34Z","completed_at":"2026-10-08T12:19:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:33:49.165648788Z","timing_metadata_checked_at":"2026-10-08T12:33:49.165648788Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:33:49.169307376Z","region_metadata_checked_at":"2026-10-08T12:33:49.169307376Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"692a210889813d7a21912c40d58e96a397f975d1867ab2bc6a21dcf8499cb894","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","failed_at":"alert","signature_id":"0505f57a41fb3f2b5cc475246473e4503d1def678146b025f446120e420b23ae","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T12:01:31Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7008768-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7008768/providers/microsoft.monitor/accounts/services-j7008768\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-58dfcbcb48-7bd6w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7008768-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T12:01:31Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7008768-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7008768/providers/microsoft.monitor/accounts/services-j7008768\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-58dfcbcb48-7bd6w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j7008768-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"c7d663428e881337b0f5e03c107a820e3c100ccd8e09c5dc5ead47fea1be95d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T10:49:03Z\n Ended: 2026-10-08T10:55:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7008768-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108129668086370304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7412,"pr_state":"closed","pr_sha":"50d28d366cb04e88caa1d494b0ac8289801425b5","final_merged_sha":"50d28d366cb04e88caa1d494b0ac8289801425b5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T09:37:34Z","started_at":"2026-10-08T09:37:34Z","completed_at":"2026-10-08T11:51:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:02:15.984769721Z","timing_metadata_checked_at":"2026-10-08T12:02:15.984769721Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:02:15.987741511Z","region_metadata_checked_at":"2026-10-08T12:02:15.987741511Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108129603208876032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"7d78fcde0fa4a260234d74d3892068a78f7d375b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T09:37:19Z","started_at":"2026-10-08T09:37:19Z","completed_at":"2026-10-08T11:55:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:12:15.54875544Z","timing_metadata_checked_at":"2026-10-08T12:12:15.54875544Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:12:15.555937421Z","region_metadata_checked_at":"2026-10-08T12:12:15.555937421Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cbcc18874d15a6a54e1b7477dc3b974662f80111d78d73e0993d387d59fb00aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108129603208876032","occurred_at":"2026-10-08T09:37:19Z","failed_at":"e2e","signature_id":"b1cc99febe4b197af72ea8a773b460d06bc2296c0c3a577338c7f8d560c7e86f","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1749b81ac3a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-jlkcb2vkhpvr\" to finish creating: GET https://rp.j8876032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdaa0297-3302-494a-9a7f-438dfbd8479c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdaa0297-3302-494a-9a7f-438dfbd8479c\",\n \"name\": \"cdaa0297-3302-494a-9a7f-438dfbd8479c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:40:01.811608698Z\",\n \"endTime\": \"2026-10-08T11:09:08.599114626Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-9xb9s-glzcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-jlkcb2vkhpvr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-9xb9s-glzcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-9xb9s-glzcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-jlkcb2vkhpvr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-9xb9s-glzcx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1749b81ac3a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-jlkcb2vkhpvr\" to finish creating: GET https://rp.j8876032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdaa0297-3302-494a-9a7f-438dfbd8479c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cdaa0297-3302-494a-9a7f-438dfbd8479c\",\n \"name\": \"cdaa0297-3302-494a-9a7f-438dfbd8479c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:40:01.811608698Z\",\n \"endTime\": \"2026-10-08T11:09:08.599114626Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-9xb9s-glzcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-jlkcb2vkhpvr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-9xb9s-glzcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-9xb9s-glzcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-jlkcb2vkhpvr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-9xb9s-glzcx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108128762271895552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"open","pr_sha":"91559182869d9c499a23720ed1842a2f04eb10f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T09:33:58Z","started_at":"2026-10-08T09:33:58Z","completed_at":"2026-10-08T11:55:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:12:15.723761065Z","timing_metadata_checked_at":"2026-10-08T12:12:15.723761065Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:12:15.726382558Z","region_metadata_checked_at":"2026-10-08T12:12:15.726382558Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3a615967ab2e6605c108efb17b764f9c406ab0575a321d7386d1681ead217d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108128762271895552","occurred_at":"2026-10-08T09:33:58Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T10:35:41Z\n Ended: 2026-10-08T10:41:38Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1895552-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108125019291783168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"open","pr_sha":"4d115c58eeb2b950c41cb99ea71cfdd98722d417","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T09:19:06Z","started_at":"2026-10-08T09:19:06Z","completed_at":"2026-10-08T11:44:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T12:02:15.588107628Z","timing_metadata_checked_at":"2026-10-08T12:02:15.588107628Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T12:02:15.591963215Z","region_metadata_checked_at":"2026-10-08T12:02:15.591963215Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108123255498870784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7420,"pr_state":"closed","pr_sha":"4f30e2ce59cc606b3bf85418f97decf933428fa1","final_merged_sha":"4f30e2ce59cc606b3bf85418f97decf933428fa1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T09:12:04Z","started_at":"2026-10-08T09:12:04Z","completed_at":"2026-10-08T11:42:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T11:52:01.059150802Z","timing_metadata_checked_at":"2026-10-08T11:52:01.059150802Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T11:52:01.065176182Z","region_metadata_checked_at":"2026-10-08T11:52:01.065176182Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"34e7ed0162d57b125ac602238ce17902497ce83e56f8c597ffa9334032de6bd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108123255498870784","occurred_at":"2026-10-08T09:12:04Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T10:15:16Z\n Ended: 2026-10-08T10:21:17Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8870784-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T09:09:57Z","started_at":"2026-10-08T09:09:57Z","completed_at":"2026-10-08T11:31:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T11:41:06.601754366Z","timing_metadata_checked_at":"2026-10-08T11:41:06.601754366Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T11:41:06.606627651Z","region_metadata_checked_at":"2026-10-08T11:41:06.606627651Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"445e134e97592b2f06db8d26a54005a571b7b73e0cae39f24243ecd90299af9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","failed_at":"e2e","signature_id":"167a399ee5497032fbc211b91c5977c4d716b98bb1d5acb6868aa44e16e76145","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39ccc1989460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-p2kczkmtdgbx\" to finish creating: GET https://rp.j5271040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f832996-ec93-465f-b6ee-4ab328dac405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f832996-ec93-465f-b6ee-4ab328dac405\",\n \"name\": \"1f832996-ec93-465f-b6ee-4ab328dac405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:21:11.725925281Z\",\n \"endTime\": \"2026-10-08T10:50:14.636681892Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zbjzt-gngqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-p2kczkmtdgbx--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zbjzt-gngqj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zbjzt-gngqj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-p2kczkmtdgbx--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zbjzt-gngqj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39ccc1989460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-p2kczkmtdgbx\" to finish creating: GET https://rp.j5271040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f832996-ec93-465f-b6ee-4ab328dac405\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f832996-ec93-465f-b6ee-4ab328dac405\",\n \"name\": \"1f832996-ec93-465f-b6ee-4ab328dac405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:21:11.725925281Z\",\n \"endTime\": \"2026-10-08T10:50:14.636681892Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zbjzt-gngqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-p2kczkmtdgbx--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zbjzt-gngqj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zbjzt-gngqj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-p2kczkmtdgbx--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zbjzt-gngqj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ecb4f4369534a4760c95ebd01cdf4937711b70e46a5b8f715767149f5afedf90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","failed_at":"e2e","signature_id":"8a3ed87cf84ff42b79a1fd8e8f951dc027047229d3c301b92c6142a018727cb3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jc5rgf\" for cluster \"cluster-candidate-4-20-jc5rgf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x49efa775a40\u003e: \n failed to create NodePool np-jc5rgf: failed waiting for nodepool=\"np-jc5rgf\" for cluster \"cluster-candidate-4-20-jc5rgf\" in resourcegroup=\"rg-candidate-4-20-cbxcrrxtz89m\" to finish creating: GET https://rp.j5271040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5aac86d-773c-4950-b645-5bbd5ad5a70a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5aac86d-773c-4950-b645-5bbd5ad5a70a\",\n \"name\": \"c5aac86d-773c-4950-b645-5bbd5ad5a70a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:23:03.365232367Z\",\n \"endTime\": \"2026-10-08T10:52:04.576957009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cbxcrrxtz89m-candidate-jc5rgf--managed/providers/Microsoft.Compute/virtualMachines/y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cbxcrrxtz89m-candidate-jc5rgf--managed/providers/Microsoft.Compute/virtualMachines/y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jc5rgf\" for cluster \"cluster-candidate-4-20-jc5rgf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x49efa775a40\u003e: \n failed to create NodePool np-jc5rgf: failed waiting for nodepool=\"np-jc5rgf\" for cluster \"cluster-candidate-4-20-jc5rgf\" in resourcegroup=\"rg-candidate-4-20-cbxcrrxtz89m\" to finish creating: GET https://rp.j5271040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5aac86d-773c-4950-b645-5bbd5ad5a70a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5aac86d-773c-4950-b645-5bbd5ad5a70a\",\n \"name\": \"c5aac86d-773c-4950-b645-5bbd5ad5a70a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:23:03.365232367Z\",\n \"endTime\": \"2026-10-08T10:52:04.576957009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cbxcrrxtz89m-candidate-jc5rgf--managed/providers/Microsoft.Compute/virtualMachines/y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cbxcrrxtz89m-candidate-jc5rgf--managed/providers/Microsoft.Compute/virtualMachines/y0h4n6k5k2x2i5v-np-jc5rgf-nhkc9-rxczq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T08:20:23Z","started_at":"2026-10-08T08:20:23Z","completed_at":"2026-10-08T11:27:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T11:41:06.796471168Z","timing_metadata_checked_at":"2026-10-08T11:41:06.796471168Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T11:41:06.799918557Z","region_metadata_checked_at":"2026-10-08T11:41:06.799918557Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5bed98507fa6a9e3be583aebc27887d48dee4ad4206305e6a8b34f423553befa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T09:56:24Z\n Ended: 2026-10-08T10:02:27Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8241280-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"8a7b8025a0a17aeee389ffd5e78b1be216108003a6f832d5157a1f560435e16a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","failed_at":"e2e","signature_id":"d6f3dbb1bf15144cc0a62926bd3e48a5cae1b7b619360a81ed11fa37f85d80e3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3a17b89ea108\u003e: \n resource group \"rg-candidate-5-0-m7pk2bqx25bp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-8cn6hj\" in resourcegroup=\"rg-candidate-5-0-m7pk2bqx25bp\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8e4a1e-473e-4006-96fb-1eb1247636a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8e4a1e-473e-4006-96fb-1eb1247636a3\",\n \"name\": \"ae8e4a1e-473e-4006-96fb-1eb1247636a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:05:41.110442631Z\",\n \"endTime\": \"2026-10-08T10:29:50.911284161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taheibf9i4u8stp8ms5pnmlpeb5go3l still exists (deletion dispatched at 2026-10-08T10:05:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3a17b89ea108\u003e: \n resource group \"rg-candidate-5-0-m7pk2bqx25bp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-8cn6hj\" in resourcegroup=\"rg-candidate-5-0-m7pk2bqx25bp\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8e4a1e-473e-4006-96fb-1eb1247636a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8e4a1e-473e-4006-96fb-1eb1247636a3\",\n \"name\": \"ae8e4a1e-473e-4006-96fb-1eb1247636a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:05:41.110442631Z\",\n \"endTime\": \"2026-10-08T10:29:50.911284161Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taheibf9i4u8stp8ms5pnmlpeb5go3l still exists (deletion dispatched at 2026-10-08T10:05:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"debce3f7d9a3353f01c47192cab1d3a796bbce92a15d8ec97e42c45356821df9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","failed_at":"e2e","signature_id":"a341e9bf98d3c6591ee21f0e758f438f31604f2f35d4323eac348c53761f7bb0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd2898b7c180\u003e: \n resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd2898b7c180\u003e: \n resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7389,"pr_state":"closed","pr_sha":"45fd914d96c1b908f204765007e479aac04b9b32","final_merged_sha":"45fd914d96c1b908f204765007e479aac04b9b32","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T08:16:51Z","started_at":"2026-10-08T08:16:51Z","completed_at":"2026-10-08T11:05:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T11:20:28.561520582Z","timing_metadata_checked_at":"2026-10-08T11:20:28.561520582Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T11:20:28.571235447Z","region_metadata_checked_at":"2026-10-08T11:20:28.571235447Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"13bdef69b97b1a9a6acefa7af8f02e0d8d0454c258e442dab1beb6f273f32884","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","failed_at":"e2e","signature_id":"ffdb06d45831bbbf8246e86634c1c7fa86a1828ef1864f846896e0d1bc90fa06","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-85g664x7dr6x/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-85g664x7dr6x/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"ba944669aa479a8b19a7d06baf920d3ed1d334fb5cb2c34ddee8c1a043a74fd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","failed_at":"e2e","signature_id":"f22d3abbd1cab4d50eb4fc487acaabfea9c0b84dab46bd31dda1987b2400c11a","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x32468342f3e0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-wtnllf\" in resourcegroup=\"rg-np-noedge-wtnllf-hjnhf58nl2hn\" to finish creating: GET https://rp.j4302720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4471bd89-63c3-4d52-ab14-11262ab0d01e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4471bd89-63c3-4d52-ab14-11262ab0d01e\",\n \"name\": \"4471bd89-63c3-4d52-ab14-11262ab0d01e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:10:30.503055567Z\",\n \"endTime\": \"2026-10-08T10:39:39.262233637Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wtnllf-hjnhf58nl2hn-np-ne-wtnllf--managed/providers/Microsoft.Compute/virtualMachines/p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wtnllf-hjnhf58nl2hn-np-ne-wtnllf--managed/providers/Microsoft.Compute/virtualMachines/p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x32468342f3e0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-wtnllf\" in resourcegroup=\"rg-np-noedge-wtnllf-hjnhf58nl2hn\" to finish creating: GET https://rp.j4302720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4471bd89-63c3-4d52-ab14-11262ab0d01e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4471bd89-63c3-4d52-ab14-11262ab0d01e\",\n \"name\": \"4471bd89-63c3-4d52-ab14-11262ab0d01e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:10:30.503055567Z\",\n \"endTime\": \"2026-10-08T10:39:39.262233637Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wtnllf-hjnhf58nl2hn-np-ne-wtnllf--managed/providers/Microsoft.Compute/virtualMachines/p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wtnllf-hjnhf58nl2hn-np-ne-wtnllf--managed/providers/Microsoft.Compute/virtualMachines/p4p6y7m3b1v5s6t-npnoedge-4-20-7bmz2-b9qj6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108105661417000960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6954,"pr_state":"open","pr_sha":"f905db4c06deef3d28a6d3e8d3a60d9d60cbd4b7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T08:01:48Z","started_at":"2026-10-08T08:01:48Z","completed_at":"2026-10-08T09:09:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T09:26:19.924933607Z","timing_metadata_checked_at":"2026-10-08T09:26:19.924933607Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T09:26:19.929003494Z","region_metadata_checked_at":"2026-10-08T09:26:19.929003494Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9bd5781e10df971e2eb891157faf239c5c691ce2dbd992f24ee0b7c7140549e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108105661417000960","occurred_at":"2026-10-08T08:01:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T07:57:36Z","started_at":"2026-10-08T07:57:36Z","completed_at":"2026-10-08T10:11:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T10:18:07.035301269Z","timing_metadata_checked_at":"2026-10-08T10:18:07.035301269Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T10:18:07.038400859Z","region_metadata_checked_at":"2026-10-08T10:18:07.038400859Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9a3fa0e57dffb01f87373ead212d04feee580a92fbdd2312c1ac7e688e0313d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","failed_at":"e2e","signature_id":"b122c2e9da305b502bb36b2c546534409e8af338dfddda201b8e60b513ad641d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bzbvdd52lfmz/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-bzbvdd52lfmz/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"dbb93f7d02e228188c712a2197784820eb2b39d807815744b2593c31331cf327","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","failed_at":"e2e","signature_id":"99e7ae876a6bc4905e749326067bbde5a6d33b2d8c46d8ed6fbac42e335e17f3","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x1409e4db1080\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x1409e4db1080\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108101483491758080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7378,"pr_state":"closed","pr_sha":"7bed7b418cbe61b464069212e28bd0e360d80b05","final_merged_sha":"7bed7b418cbe61b464069212e28bd0e360d80b05","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T07:45:34Z","started_at":"2026-10-08T07:45:34Z","completed_at":"2026-10-08T10:11:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T10:18:07.332572694Z","timing_metadata_checked_at":"2026-10-08T10:18:07.332572694Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T10:18:07.335173186Z","region_metadata_checked_at":"2026-10-08T10:18:07.335173186Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"87bcc3e2b604ee3ac45f7d2dbc4d89d48f608c8a8fc017a237518fac3d71e5a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108101483491758080","occurred_at":"2026-10-08T07:45:34Z","failed_at":"alert","signature_id":"afb48a8584f79bbdd2eee32217731be00e00c63d7874b7427fe12a549c0c7d4a","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T09:47:35Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1758080-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.67:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1758080/providers/microsoft.monitor/accounts/services-j1758080\", namespace=\"klusterlet-2tage7jjs69ome454gjasrbup2nm9mab\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-whw96\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tage7jjs69ome454gjasrbup2nm9mab on cluster ci01-j1758080-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108100160046239744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6092,"pr_state":"closed","pr_sha":"6c52f2825f7e760873c725bfc4347e864840e655","final_merged_sha":"6c52f2825f7e760873c725bfc4347e864840e655","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T07:40:19Z","started_at":"2026-10-08T07:40:19Z","completed_at":"2026-10-08T09:59:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T10:18:06.827967549Z","timing_metadata_checked_at":"2026-10-08T10:18:06.827967549Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T10:18:06.831785436Z","region_metadata_checked_at":"2026-10-08T10:18:06.831785436Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7412,"pr_state":"closed","pr_sha":"50d28d366cb04e88caa1d494b0ac8289801425b5","final_merged_sha":"50d28d366cb04e88caa1d494b0ac8289801425b5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T07:10:43Z","started_at":"2026-10-08T07:10:43Z","completed_at":"2026-10-08T09:31:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T09:36:36.801962266Z","timing_metadata_checked_at":"2026-10-08T09:36:36.801962266Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T09:36:36.804763657Z","region_metadata_checked_at":"2026-10-08T09:36:36.804763657Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"8a53cafdf775520cecd10f901bb3c3b7e544c12950d664eed661f9e16c7e7902","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","failed_at":"alert","signature_id":"17502e6a782e32cf8d05639323afa25f0eb69d1c67087752a4cd2f50e385cc87","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T08:59:55Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9632000-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.149:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9632000/providers/microsoft.monitor/accounts/services-j9632000\", namespace=\"klusterlet-2tafv3kk3g55204rl52vhttrl8vlq1g0\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-hq5xt\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tafv3kk3g55204rl52vhttrl8vlq1g0 on cluster ci01-j9632000-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"b219fe2ca92af1c34525ff30b17cc1c390c8e0543ae4a9aed4211596ea2f202d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","failed_at":"e2e","signature_id":"99a86419200aac49952e3148c95bd65fe8b3fbe3e68a7b83763b8f9aa7967171","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e0336d1d382f00c7472dffb932a4ea9f4579aa5863de6407d8639790490c2c1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","failed_at":"e2e","signature_id":"5fc986fabe0026f4b77c8908e8c0e313a1116526b8618c4e0b755089134b6b09","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xda1a20062a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-lbppg5gfcqw8\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f1d411e-4147-426c-8a46-dece07469237\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f1d411e-4147-426c-8a46-dece07469237\",\n \"name\": \"5f1d411e-4147-426c-8a46-dece07469237\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:09:00.258803996Z\",\n \"endTime\": \"2026-10-08T08:38:02.86327598Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zz94m-k596d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-lbppg5gfcqw8--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zz94m-k596d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zz94m-k596d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-lbppg5gfcqw8--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zz94m-k596d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xda1a20062a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-lbppg5gfcqw8\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f1d411e-4147-426c-8a46-dece07469237\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f1d411e-4147-426c-8a46-dece07469237\",\n \"name\": \"5f1d411e-4147-426c-8a46-dece07469237\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:09:00.258803996Z\",\n \"endTime\": \"2026-10-08T08:38:02.86327598Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zz94m-k596d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-lbppg5gfcqw8--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zz94m-k596d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zz94m-k596d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-lbppg5gfcqw8--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zz94m-k596d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7419/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108091111233818624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7419,"pr_state":"closed","pr_sha":"cb2a2f44f515d0152591fc7b1539b79c936ba059","final_merged_sha":"cb2a2f44f515d0152591fc7b1539b79c936ba059","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T07:04:21Z","started_at":"2026-10-08T07:04:21Z","completed_at":"2026-10-08T09:41:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T09:46:39.706351981Z","timing_metadata_checked_at":"2026-10-08T09:46:39.706351981Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T09:46:39.709664169Z","region_metadata_checked_at":"2026-10-08T09:46:39.709664169Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7325,"pr_state":"closed","pr_sha":"d33f2036e296405613325656050c42e37791249d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T07:02:22Z","started_at":"2026-10-08T07:02:22Z","completed_at":"2026-10-08T09:18:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T09:26:20.109564112Z","timing_metadata_checked_at":"2026-10-08T09:26:20.109564112Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T09:26:20.112664302Z","region_metadata_checked_at":"2026-10-08T09:26:20.112664302Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"221a28dfb205c783c075f7820c1745c86b5b10edf8bd05027a1c8b9132e45597","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","failed_at":"e2e","signature_id":"06adbd5f174da4a722aae0d056ec2b5a7c54b016aa9e689d5cee91e57df2195d","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"f14428a1aaa6d1bc0585fb9442c61c3562c25795d05249c19cfeececf332fc27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","failed_at":"e2e","signature_id":"3cc6e00f4b908acb0ea62e41044f8a94e6ce5698be960fa38a7f85470bd35cba","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7415/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108062856111984640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7415,"pr_state":"closed","pr_sha":"69cfc5f9ad80ba66550f6abcb23a724ea7c42846","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T05:12:03Z","started_at":"2026-10-08T05:12:03Z","completed_at":"2026-10-08T07:43:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T08:02:05.613193229Z","timing_metadata_checked_at":"2026-10-08T08:02:05.613193229Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T08:02:05.617971113Z","region_metadata_checked_at":"2026-10-08T08:02:05.617971113Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T04:44:42Z","started_at":"2026-10-08T04:44:42Z","completed_at":"2026-10-08T07:06:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T07:09:32.609887679Z","timing_metadata_checked_at":"2026-10-08T07:09:32.609887679Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T07:09:32.612881769Z","region_metadata_checked_at":"2026-10-08T07:09:32.612881769Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"4a196792ede61b9aed2cd0cac20bc35314c41945511f04e62b48facfe419cd54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"alert","signature_id":"7e1b30717a7056324794286c92396616dc0d26090203314168033730260fac49","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T06:09:36Z\n Ended: 2026-10-08T06:23:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-g29zr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5b0d0346cea8c72beabf0832f38c0fc081c3d84345bc5c046df0f1650c07529b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"alert","signature_id":"85682e6aa72ffd8e2527c0a0e4f27430fd467da3643a75af116ad2157337fe40","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T05:54:34Z\n Ended: 2026-10-08T06:18:34Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-g29zr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-08T05:54:34Z\n Ended: 2026-10-08T06:19:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-g29zr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"7c1a8a1c4ced45a4f2c2d6eb8cd05e112aca2f2da1f1ea4c362a8761243dc606","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"alert","signature_id":"e76d176cb58c095e589bc218c3e104c756d14a6ceb09fa88d761f62736c4f607","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T06:00:31Z\n Ended: 2026-10-08T06:22:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9e83f444-711c-486c-b496-8152e0711540\"\n Description: Pod ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-08T06:00:31Z\n Ended: 2026-10-08T06:22:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9e83f444-711c-486c-b496-8152e0711540\"\n Description: Pod ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"9b2bf07937c2e0a2c0888748b5741e63d441439856270ec22a710a26cdfc5a13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T05:48:26Z\n Ended: 2026-10-08T06:05:24Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4542976-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"cb56e9983b4a8e844ac61ea7d747d1d05264774af48ae5a7fccf8c387249694e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"e2e","signature_id":"b132c43fd39d685c72ce0a5f22659a72c1f81a5d0a9dd5f857638ee06b041704","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-zj96vcc7brng/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-zj96vcc7brng/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"df2c64a2c9ac844e451f99d4892aa1a91ab8cc267adac0dab8cda632e88aeed5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","failed_at":"e2e","signature_id":"09cb0962695c90bbcb8aa045ebb144f397afbd0455023f7fea0160df7e60cafc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xa118982a360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dx54wv2l48hl\" to finish creating: GET https://rp.j4542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"name\": \"54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:35:12.262324182Z\",\n \"endTime\": \"2026-10-08T06:04:17.39866058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xa118982a360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dx54wv2l48hl\" to finish creating: GET https://rp.j4542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"name\": \"54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:35:12.262324182Z\",\n \"endTime\": \"2026-10-08T06:04:17.39866058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"open","pr_sha":"9ea305d11e1b7326426345a8070e9f58dd45e419","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T03:54:28Z","started_at":"2026-10-08T03:54:28Z","completed_at":"2026-10-08T06:17:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T06:37:07.067291985Z","timing_metadata_checked_at":"2026-10-08T06:37:07.067291985Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T06:37:07.071214673Z","region_metadata_checked_at":"2026-10-08T06:37:07.071214673Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"238286dea43191803df46237c22eb833dc4670aa2317ba711c755580c5160f9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","failed_at":"e2e","signature_id":"f21ba692b551caebe9fb47bce79bd94b5286f66a20a64edd5fe17a6b67fb55c5","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x32ec947f60c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-mkxhsl7j2nvb\" to finish creating: GET https://rp.j2143872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2ab5edd-253e-4c79-8950-79decb992f4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2ab5edd-253e-4c79-8950-79decb992f4b\",\n \"name\": \"b2ab5edd-253e-4c79-8950-79decb992f4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:56:23.95109392Z\",\n \"endTime\": \"2026-10-08T05:25:32.665732941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zg8r4-bhhzf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-mkxhsl7j2nvb--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zg8r4-bhhzf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zg8r4-bhhzf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-mkxhsl7j2nvb--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zg8r4-bhhzf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x32ec947f60c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-mkxhsl7j2nvb\" to finish creating: GET https://rp.j2143872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2ab5edd-253e-4c79-8950-79decb992f4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2ab5edd-253e-4c79-8950-79decb992f4b\",\n \"name\": \"b2ab5edd-253e-4c79-8950-79decb992f4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:56:23.95109392Z\",\n \"endTime\": \"2026-10-08T05:25:32.665732941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-zg8r4-bhhzf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-mkxhsl7j2nvb--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zg8r4-bhhzf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-zg8r4-bhhzf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-mkxhsl7j2nvb--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-zg8r4-bhhzf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7126ca94825bc1d3a420d1fa4668999b52d90dcd51a71391345e4a18da07f917","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T05:01:06Z\n Ended: 2026-10-08T05:07:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2143872-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7412,"pr_state":"closed","pr_sha":"b4d45119daa5446187f3eb2feefc6cd8958065ae","final_merged_sha":"50d28d366cb04e88caa1d494b0ac8289801425b5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T03:07:29Z","started_at":"2026-10-08T03:07:29Z","completed_at":"2026-10-08T05:16:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T05:32:44.193431124Z","timing_metadata_checked_at":"2026-10-08T05:32:44.193431124Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T05:32:44.196352813Z","region_metadata_checked_at":"2026-10-08T05:32:44.196352813Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"571255690957f5ede925c89ee935bfe92c184a2136904f8ba9b2b9686868a9f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T04:04:43Z\n Ended: 2026-10-08T04:10:44Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5015552-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"fa468174821a145fa7a43407c23d2fc0176d3413168d0613dd8bca5a16c7293f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","failed_at":"alert","signature_id":"33a563d1f9854fc5c2d805dce229ec37f3683cf9f20862a546be44d8addf4f52","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 57 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-gvdlf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-gvdlf has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-pj78n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-pj78n has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-jnjtd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-jnjtd has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rplxb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rplxb has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-htmxm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-htmxm has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rv6cc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rv6cc has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"cloud-network-config-controller-74769fccfb-k8kwp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/cloud-network-config-controller-74769fccfb-k8kwp has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-hgjs7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-hgjs7 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-gs5jc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-gs5jc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-8chb5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-8chb5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-psbgs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-psbgs has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-4tbx4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-4tbx4 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-gjpp8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-gjpp8 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-j44kk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-j44kk has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-mvbj5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-mvbj5 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-ss6rg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-ss6rg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-snf5p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-snf5p has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-mdgsc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-mdgsc has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-cwkvx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-cwkvx has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-bnmrb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-bnmrb has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-cv4g6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-cv4g6 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-7n652\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-7n652 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-4hlqb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-4hlqb has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-w2spp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-w2spp has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-87cjv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-87cjv has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-ht9c9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-ht9c9 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-jwnrm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-jwnrm has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-8mmlh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-8mmlh has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-7pzw8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-7pzw8 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-672v2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-672v2 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-9c8996664-xqlcz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-9c8996664-xqlcz has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-f8gbx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-f8gbx has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-l29xs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-l29xs has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-mcs4j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-mcs4j has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-m8sbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-m8sbd has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-fg868\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-fg868 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-d57l7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-d57l7 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-6qjzf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-6qjzf has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-ssj25\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-ssj25 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2nvzx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2nvzx has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-4rxvd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-4rxvd has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-l6dz6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-l6dz6 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-6qjld\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-6qjld has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-m9rtt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-m9rtt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2njx7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2njx7 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2t8lh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2t8lh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rhccp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rhccp has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-g64ns\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-g64ns has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-5c9wj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-5c9wj has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-kdtjv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-kdtjv has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-5fld5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-5fld5 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-skdmq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-skdmq has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-lfqpn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-lfqpn has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-kq58q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-kq58q has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-pjnm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-pjnm6 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-hztch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-hztch has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-zgk6p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-zgk6p has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108026812570275840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:48:51Z","started_at":"2026-10-08T02:48:51Z","completed_at":"2026-10-08T03:46:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:57:53.80005761Z","timing_metadata_checked_at":"2026-10-08T03:57:53.80005761Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T03:57:53.805877291Z","region_metadata_checked_at":"2026-10-08T03:57:53.805877291Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"64e0dfb46f00e9aa9ce1d1eed18a1c1c402d2e5d37f2e21674aa5a12a120e514","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108026812570275840","occurred_at":"2026-10-08T02:48:51Z","failed_at":"provision","signature_id":"6bac7c9167143bb4a1dfabfc4a6d81083bef0f715dcabad843cd680ef59999d4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T03:31:09.066Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpci01 kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:c5d026c4a8335c2d098ce04776e25babc3e7773e70817b5b709f388bb52488d2] metricLabelsAllowlist:[nodes=[node.kubernetes.io/instance-type,kubernetes.azure.com/agentpool]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[system]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:9c285c83d939df7d0bdf27287562bb6fcf8fd6ab639d899abba1f14a0c2d5647] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:5fc5d09d0245cc1e7c6432ca9b0ebcc652c047b550d0289ea45725c14343585d]] thanosImage:map[registry:arohcpsvcdev.azurecr.io repository:thanos/thanos sha:b567818fe608067eb0f1d7c2c4fe361e7ad83c8a256234c97685f1d0bf670cc8 tag:v0.41.0] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[serviceAccount:map[managedIdentity:0b00177b-a54e-4160-b95f-d8ba55b89f42]] prometheusSpec:map[externalLabels:map[cluster:ci01-j0275840-svc environment:ci01 region:westus3] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:7ef9c1d80e4e8e1eec1a207166af22930c8f11200f25001d1a3285b30b4f13f9] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-ci01-j0275840-svc-ng1v.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d00da46377ca403ab27fbe4e13d51efe/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 replicas:2 resources:map[limits:map[memory:NONE] requests:map[cpu:NONE memory:NONE]] shards:1 version: zoneCount:3] schedulingRole:system]\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.214Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.783Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478","failure_pattern":"failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025152309563392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:42:16Z","started_at":"2026-10-08T02:42:16Z","completed_at":"2026-10-08T03:43:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:57:54.219591728Z","timing_metadata_checked_at":"2026-10-08T03:57:54.219591728Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T03:57:54.222685418Z","region_metadata_checked_at":"2026-10-08T03:57:54.222685418Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"48dc0afea1ed23d0d0562003bba73d911dea47aa0e2320b7b3305505a626dbee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025152309563392","occurred_at":"2026-10-08T02:42:16Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-08T03:34:00.150Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-08T03:34:00.987Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-08T03:34:02.023Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9563392-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220\ntime=2026-10-08T03:36:02.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220/operationStatuses/08584101764439358601\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mgmt-agent' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1a2d3dadcb1c305fd461d5e9526c1e1f98093bc5d6545af22d4aa0cf7e7d759","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d1a2d3dadcb1c305fd461d5e9526c1e1f98093bc5d6545af22d4aa0cf7e7d759"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025105396273152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7288,"pr_state":"closed","pr_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","final_merged_sha":"37eef9bef9919c0c3e976b60d3658d6356dcb4f1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:42:04Z","started_at":"2026-10-08T02:42:04Z","completed_at":"2026-10-08T05:14:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T05:32:44.167780922Z","timing_metadata_checked_at":"2026-10-08T05:32:44.167780922Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T05:32:44.17625399Z","region_metadata_checked_at":"2026-10-08T05:32:44.17625399Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2a3858039af12252ba95009678ecb843799e8050ba9d1b69262b0cc0deb1e1f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025105396273152","occurred_at":"2026-10-08T02:42:04Z","failed_at":"e2e","signature_id":"b6ae0ddb16ff4c94221800e11be0c27c614a44b85087bb2bbb64f1a59d3dee88","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n {\n errs: [\n \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n Reason: \"InternalError\",\n Details: {\n Name: \"fips-check-9c9j4\",\n Group: \"\",\n Kind: \"pods\",\n UID: \"\",\n Causes: [\n {Type: ..., Message: ..., Field: ...},\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n {\n errs: [\n \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n Reason: \"InternalError\",\n Details: {\n Name: \"fips-check-9c9j4\",\n Group: \"\",\n Kind: \"pods\",\n UID: \"\",\n Causes: [\n {Type: ..., Message: ..., Field: ...},\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025089978011648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7361,"pr_state":"closed","pr_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","final_merged_sha":"977f1d7319139af73f9cb29012b400bc9def5a39","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:42:01Z","started_at":"2026-10-08T02:42:01Z","completed_at":"2026-10-08T05:09:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T05:22:08.846696284Z","timing_metadata_checked_at":"2026-10-08T05:22:08.846696284Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T05:22:08.850566071Z","region_metadata_checked_at":"2026-10-08T05:22:08.850566071Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3ba16e14988b5fd33c6bc27a09705620c217003ed8a607f83fa0e238920bc9fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025089978011648","occurred_at":"2026-10-08T02:42:01Z","failed_at":"e2e","signature_id":"898639e46d512657722f4281a7d0dfb4428d8ed1240fc047c4055963519f6d5a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc4b0a06e918\u003e: \n failed to create NodePool agg-np-b: failed waiting for nodepool=\"agg-np-b\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-qnvb5555k8tb\" to finish creating: GET https://rp.j8011648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"name\": \"7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:02:19.757714703Z\",\n \"endTime\": \"2026-10-08T04:31:24.163355482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc4b0a06e918\u003e: \n failed to create NodePool agg-np-b: failed waiting for nodepool=\"agg-np-b\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-qnvb5555k8tb\" to finish creating: GET https://rp.j8011648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"name\": \"7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:02:19.757714703Z\",\n \"endTime\": \"2026-10-08T04:31:24.163355482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7325,"pr_state":"closed","pr_sha":"eba089832a07cda12e680d1790409cccb24991c1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T02:14:25Z","started_at":"2026-10-08T02:14:25Z","completed_at":"2026-10-08T04:24:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T04:39:48.326148821Z","timing_metadata_checked_at":"2026-10-08T04:39:48.326148821Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T04:39:48.329192313Z","region_metadata_checked_at":"2026-10-08T04:39:48.329192313Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"04fae0da2ba8ea0acd569615190ba14b49e15fb2ae1dd672633897b21fd13737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T03:22:03Z\n Ended: 2026-10-08T03:28:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2503552-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c0919bcfec4263fd5b6cd779222c25586dc6097fea6d18c88c4fafa3758320f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","failed_at":"e2e","signature_id":"dc991bbf770e80e3e153a619922727dc6f068b8152a0521def1711d4a491b244","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"f39063971388a80d198990bcd5f9942970bc44d8721a3427683d9641f4a44cbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","failed_at":"e2e","signature_id":"38cafc0c9d53e48809e4f80070514d02cab460de26c498c2ee86e36255eb9c93","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108015628659986432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"3eb0d9589874d6e07d11073f2d1159d31973c7e0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-08T02:04:25Z","started_at":"2026-10-08T02:04:25Z","completed_at":"2026-10-08T04:22:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T04:39:48.136016069Z","timing_metadata_checked_at":"2026-10-08T04:39:48.136016069Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T04:39:48.145317142Z","region_metadata_checked_at":"2026-10-08T04:39:48.145317142Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"95bbbe31e06e88585eccdad7152ce6792fa23562c94d7a2cdeb2aa25ebf31842","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108015628659986432","occurred_at":"2026-10-08T02:04:25Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T03:04:48Z\n Ended: 2026-10-08T03:10:47Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9986432-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7417/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108005201297281024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7417,"pr_state":"closed","pr_sha":"370e1ce005aa6305e203298ced0a3bdd3f407900","final_merged_sha":"370e1ce005aa6305e203298ced0a3bdd3f407900","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T01:22:59Z","started_at":"2026-10-08T01:22:59Z","completed_at":"2026-10-08T03:25:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:36:52.356982185Z","timing_metadata_checked_at":"2026-10-08T03:36:52.356982185Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T03:36:52.36167987Z","region_metadata_checked_at":"2026-10-08T03:36:52.36167987Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108002043938476032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7400,"pr_state":"open","pr_sha":"f230ee1c04f30e828fc70c56d96c6582993882df","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-08T01:10:26Z","started_at":"2026-10-08T01:10:26Z","completed_at":"2026-10-08T03:39:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:57:54.022958475Z","timing_metadata_checked_at":"2026-10-08T03:57:54.022958475Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T03:57:54.026181965Z","region_metadata_checked_at":"2026-10-08T03:57:54.026181965Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108334615943974912","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T23:11:58Z","started_at":"2026-10-08T23:11:58Z","completed_at":"2026-10-09T01:23:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T01:41:40.048555182Z","timing_metadata_checked_at":"2026-10-09T01:41:40.048555182Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T01:41:40.056884355Z","region_metadata_checked_at":"2026-10-09T01:41:40.056884355Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108289457667969024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T20:12:31Z","started_at":"2026-10-08T20:12:31Z","completed_at":"2026-10-08T23:04:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:07:02.586049207Z","timing_metadata_checked_at":"2026-10-08T23:07:02.586049207Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:07:02.589188598Z","region_metadata_checked_at":"2026-10-08T23:07:02.589188598Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"33322c9eecac71a00d5695835c0886c0058558f83c914f3b48ee334d81841da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108289457667969024","occurred_at":"2026-10-08T20:12:31Z","failed_at":"e2e","signature_id":"4a13b0e3b632754c621cd1f40992e4f49a86b9b5a5adaf7a0291091d3c763eaa","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108259102915301376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T18:11:54Z","started_at":"2026-10-08T18:11:54Z","completed_at":"2026-10-08T21:06:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:11:10.594225277Z","timing_metadata_checked_at":"2026-10-08T21:11:10.594225277Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T21:11:10.598475065Z","region_metadata_checked_at":"2026-10-08T21:11:10.598475065Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T15:43:42Z","started_at":"2026-10-08T15:43:42Z","completed_at":"2026-10-08T18:58:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:03:59.473658863Z","timing_metadata_checked_at":"2026-10-08T19:03:59.473658863Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:03:59.478913946Z","region_metadata_checked_at":"2026-10-08T19:03:59.478913946Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"15f6518f2bf61dfd3ea93b0c5de1b280d5a636d81d3718e17d5b6f3a4af16ed9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"c37035f44e6baf132cd362aa9454d42aa8651a4d8c8d859449fd5863866a8dc3","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n {\n msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n {\n msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded"}},{"row_id":"3c9b4187a3ac94829ee48978334868fc49174537d57315f5c6a841bb63aa59df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"7801a21b4f98263d138c84dd4d41be41bec31d075db9a5534f6fe7a256d4fe97","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x317d01c40258\u003e: \n resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x317d01c40258\u003e: \n resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ce576e5622c2cd15b9fa8b19a44b4c91c456739b9a616c0260a5c3dbd1524dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"afaf794a0f74bf5028bd0c8acfda702e8fa09c3b57b33581a270af808e618e1a","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x19f9b5e0a3f0\u003e: \n resource group \"customer-rg-wkldzssbjxmj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wkldzssbjxmj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n \"name\": \"eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:08:36.973340805Z\",\n \"endTime\": \"2026-10-08T17:32:41.526425453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tannaefijum1ouk23vffl0tmfuhorgq still exists (deletion dispatched at 2026-10-08T17:08:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x19f9b5e0a3f0\u003e: \n resource group \"customer-rg-wkldzssbjxmj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wkldzssbjxmj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n \"name\": \"eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:08:36.973340805Z\",\n \"endTime\": \"2026-10-08T17:32:41.526425453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tannaefijum1ouk23vffl0tmfuhorgq still exists (deletion dispatched at 2026-10-08T17:08:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b202cd2ceed97d85f49097aa18f27a043a6ccabc399c0010d0511712d6e27c91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"2bccea3f5889cc8d014f896b80d99afeb5d988c9e408682319482453975203f2","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x63eda6241c8\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x63eda6241c8\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108055343585038336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T04:42:14Z","started_at":"2026-10-08T04:42:14Z","completed_at":"2026-10-08T07:16:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T07:20:00.897066011Z","timing_metadata_checked_at":"2026-10-08T07:20:00.897066011Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T07:20:00.911399464Z","region_metadata_checked_at":"2026-10-08T07:20:00.911399464Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T00:15:03Z","started_at":"2026-10-08T00:15:03Z","completed_at":"2026-10-08T02:30:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T02:34:12.942663369Z","timing_metadata_checked_at":"2026-10-08T02:34:12.942663369Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T02:34:12.947766951Z","region_metadata_checked_at":"2026-10-08T02:34:12.947766951Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2bcee9e4c62bb8bf3094b01035a9d56fc62223c3e5ea60de974fe9f92ad2a4a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","failed_at":"e2e","signature_id":"56eea831600730567aa0c1ce05d30226ca8a8d1e83b1c5e0f2bc75bde312d724","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2700c25b4198\u003e: \n resource group \"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wvvvhw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n \"name\": \"cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:41:07.332810869Z\",\n \"endTime\": \"2026-10-08T02:05:10.961421785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa15pl8rhqv7goa573mk2hebqeei6j still exists (deletion dispatched at 2026-10-08T01:41:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2700c25b4198\u003e: \n resource group \"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wvvvhw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n \"name\": \"cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:41:07.332810869Z\",\n \"endTime\": \"2026-10-08T02:05:10.961421785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa15pl8rhqv7goa573mk2hebqeei6j still exists (deletion dispatched at 2026-10-08T01:41:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60cbfcc093cc71a327f97e4555f0ac485556a218088fbd729807cbe318944606","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","failed_at":"e2e","signature_id":"4687436a0bfcfca8d25b9c7ea62a3459ab48c2ddd70a922beb77b67c78af05b7","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1a2d62cf4c18\u003e: \n resource group \"node-limit-test-7ngnq4mvtzfd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7ngnq4mvtzfd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\",\n \"name\": \"7157c234-f4b7-4732-ad4a-89009867cbd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:35:15.90215739Z\",\n \"endTime\": \"2026-10-08T01:59:21.185260922Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa7pjbbs0aafhd6esu75djenuq7kvc still exists (deletion dispatched at 2026-10-08T01:35:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1a2d62cf4c18\u003e: \n resource group \"node-limit-test-7ngnq4mvtzfd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7ngnq4mvtzfd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\",\n \"name\": \"7157c234-f4b7-4732-ad4a-89009867cbd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:35:15.90215739Z\",\n \"endTime\": \"2026-10-08T01:59:21.185260922Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa7pjbbs0aafhd6esu75djenuq7kvc still exists (deletion dispatched at 2026-10-08T01:35:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"prod","summary":{"total_runs":7,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T23:00:38Z","started_at":"2026-10-08T23:00:38Z","completed_at":"2026-10-09T00:48:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:59:36.221154786Z","timing_metadata_checked_at":"2026-10-09T00:59:36.221154786Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:59:36.224861276Z","region_metadata_checked_at":"2026-10-09T00:59:36.224861276Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"206d5abc154cdb0ba32661e26002d4ca954ba241fb0d74fd519f149748b75aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"fc06ea34826977c15833fec7d2cf3a0aa68559a73c0a34b4f46fe7992c958855","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x1bc3bcaaef00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8cplvshb5svv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x1bc3bcaaef00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8cplvshb5svv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4dd8639353e2e29ca870cbf053e369705a441d2dee2631e28520f138327aef81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"583203b05397b372a4354f60b57a0bc62bc0edfa59d540e09c922f53a5450367","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x17466b892d50\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n8q8j5xgxdk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x17466b892d50\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n8q8j5xgxdk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6927a42ae38084140fc26f8cf96613c6437635b5c206d211016b481a9cc39079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"948425aa0583e35245acd6dfa26e29e48e09e7387b7c14f662b8e630e477a7b4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2e62773595c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-m97nzsnh6v7k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2e62773595c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-m97nzsnh6v7k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6a5ffb775a61db9199bbcc0e043e00fefa37c972db5b1c1de62a03114ed96b50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"538bc1de69b7243ea254b7eafc69d03d17567c9e64b692cd74fb0c6457d5cc48","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ea6fe4464b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rwz8zs2wwdcn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ea6fe4464b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rwz8zs2wwdcn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e1bcf6a02996f90595a0f9f4e4a9241d2b9ff98334f6b193cbdd8abe79433fa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"a9cdc35fe4bd7f2b83c85c8090894982dc2b8c173bdc03255cf86269d22ffc5b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d03e6b69a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vv42cndqzc2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d03e6b69a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vv42cndqzc2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108305902917390336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T21:17:52Z","started_at":"2026-10-08T21:17:52Z","completed_at":"2026-10-08T23:30:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:48:24.543521521Z","timing_metadata_checked_at":"2026-10-08T23:48:24.543521521Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:48:24.547289309Z","region_metadata_checked_at":"2026-10-08T23:48:24.547289309Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108296013604196352","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T20:38:34Z","started_at":"2026-10-08T20:38:34Z","completed_at":"2026-10-08T21:09:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:11:10.865452063Z","timing_metadata_checked_at":"2026-10-08T21:11:10.865452063Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-08T21:11:10.868361155Z","region_metadata_checked_at":"2026-10-08T22:13:41.860924936Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ba528236be51c71c31fa625d3952e53e5a17edf1d0651b7f1c8ec6d76513d71b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108296013604196352","occurred_at":"2026-10-08T20:38:34Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T17:28:22Z","started_at":"2026-10-08T17:28:22Z","completed_at":"2026-10-08T20:12:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:28:40.945345149Z","timing_metadata_checked_at":"2026-10-08T20:28:40.945345149Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:28:40.949121438Z","region_metadata_checked_at":"2026-10-08T20:28:40.949121438Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0d087ab08155503f2bc663360b9ff216aa5e408a20ab7a902033010f008503a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"e57178da6231f676c3543909d6927a0653c8f12c71f9a8924178794a486457a8","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7b26aac4c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fbkb2qz6f9rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n \"name\": \"9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:03:48.271009954Z\",\n \"endTime\": \"2026-10-08T18:32:55.449378647Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e7b26aac4c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fbkb2qz6f9rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n \"name\": \"9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:03:48.271009954Z\",\n \"endTime\": \"2026-10-08T18:32:55.449378647Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"22572715b80e8017a8994f225991bfe917489f96bf2dfda9399609f30eb9bc63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"cbd310b6320cdff663edcfcdc2aaf12bce0f77d46d8accdace4e10db5e71773a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18c3c8280a0\u003e: \n failed to create NodePool np-cz595b: failed waiting for nodepool=\"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\" in resourcegroup=\"rg-candidate-4-22-vzqcrpffcth2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n \"name\": \"7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:36:25.022726186Z\",\n \"endTime\": \"2026-10-08T19:05:25.609844949Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18c3c8280a0\u003e: \n failed to create NodePool np-cz595b: failed waiting for nodepool=\"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\" in resourcegroup=\"rg-candidate-4-22-vzqcrpffcth2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n \"name\": \"7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:36:25.022726186Z\",\n \"endTime\": \"2026-10-08T19:05:25.609844949Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"61d521d0066674c9556d52aff325a12fbf9f5c2e2ebd8df1aa05ea70f1b6f1cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"9fc1a6a4fa883fc6875002b44e541ced39ee5185ee8b663d0df77575e08f0939","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1aded3148440\u003e: \n failed to create NodePool np-fn9nq4: failed waiting for nodepool=\"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\" in resourcegroup=\"rg-candidate-4-21-ghsd6g8zdnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\",\n \"name\": \"70a9d05d-3834-496b-ab23-90c8635291e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:35:08.455675057Z\",\n \"endTime\": \"2026-10-08T19:04:15.635949535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1aded3148440\u003e: \n failed to create NodePool np-fn9nq4: failed waiting for nodepool=\"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\" in resourcegroup=\"rg-candidate-4-21-ghsd6g8zdnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\",\n \"name\": \"70a9d05d-3834-496b-ab23-90c8635291e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:35:08.455675057Z\",\n \"endTime\": \"2026-10-08T19:04:15.635949535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6bcc7eb7e1f4e05a49d8aa4dd6e694939e625c1fbb505d3b5b17ce4e399b7bac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"30be14ccfca5a103a44375ac3df669c521021f4c93308b56a32de7bd14926922","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"8645bfeb502dfcfc6d12093b2e514747378f78d09209b06c529eba6da0daa6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"607737456265c716166bec4f750288f0594652400fdb685dc45a9ab0c9b032ba","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b97b4c2d05d9dde28f9d1adfac53413e96d6c5f7db4de9e63a56c57a88dd4e41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"ecfced95d3fefee4d4fd692bafa6b115f8cb1c09453e7e1b0e30620a04bb8554","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3c50440e0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3c50440e0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"cb34c905d3813c4e6a2b253a589bddf1eaaac5b852a44074acf7a5fba8e73b42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"eae823e6b74a95a2b1e84a019161309a32d30986fbaa5b35e10457bb32383c4a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b65d70d0400\u003e: \n failed to create NodePool np-dlrjwb: failed waiting for nodepool=\"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\" in resourcegroup=\"rg-candidate-4-23-sdtvdwbjbxfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\",\n \"name\": \"79fc773d-2a37-4106-8f63-f194f928388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:36:39.156679392Z\",\n \"endTime\": \"2026-10-08T19:05:45.645587063Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b65d70d0400\u003e: \n failed to create NodePool np-dlrjwb: failed waiting for nodepool=\"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\" in resourcegroup=\"rg-candidate-4-23-sdtvdwbjbxfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\",\n \"name\": \"79fc773d-2a37-4106-8f63-f194f928388d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:36:39.156679392Z\",\n \"endTime\": \"2026-10-08T19:05:45.645587063Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"da5efc0de8d584adc37d33c591b14f55637bbf91d135a302625dcfd3958a29e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"1de706bd8c752817d789e3f2ee0485ef15ce188abeabb1b4b45baa1c1e902769","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"daea0d8bed6d012decb218d46bc677e81b282c11fb9a160400966d42a17c7225","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"8360fc4f2965ead66c616d75f5409d9c8aee0164b8d4d71a8ba9d6db0fc9f8aa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f47cc0aa84d55fc1eeeccb754a02990577c53a17bfbed3403966cca213c39e66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"8fae7edbf3fe4eba4c8d3ad2ab08abf901803a5c7eb69c8c4b81e4c6355c2ac1","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x237e16e43180\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-q28hc5z2kwsh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n \"name\": \"6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:03:48.204928396Z\",\n \"endTime\": \"2026-10-08T18:32:55.458196493Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-j8vmw-6lgjc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-j8vmw-6lgjc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x237e16e43180\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-q28hc5z2kwsh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n \"name\": \"6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T18:03:48.204928396Z\",\n \"endTime\": \"2026-10-08T18:32:55.458196493Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-j8vmw-6lgjc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-j8vmw-6lgjc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T13:52:10Z","started_at":"2026-10-08T13:52:10Z","completed_at":"2026-10-08T16:34:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:46.527369524Z","timing_metadata_checked_at":"2026-10-08T16:44:46.527369524Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:46.530695513Z","region_metadata_checked_at":"2026-10-08T16:44:46.530695513Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"608d32b47689dce6489539b93d853020c1792a101efd104a40762f168623ad07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"9b452c1be3c93f975397b5ccfaabffaec5e3ee43b442f86ff7704c7cfc6c3d23","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39e342643720\u003e: \n failed to create NodePool np-qzm2gk: failed waiting for nodepool=\"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\" in resourcegroup=\"rg-candidate-4-21-t5wksgdrrckp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n \"name\": \"afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:53:33.939084658Z\",\n \"endTime\": \"2026-10-08T15:22:35.324122041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39e342643720\u003e: \n failed to create NodePool np-qzm2gk: failed waiting for nodepool=\"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\" in resourcegroup=\"rg-candidate-4-21-t5wksgdrrckp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n \"name\": \"afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:53:33.939084658Z\",\n \"endTime\": \"2026-10-08T15:22:35.324122041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c516bd6cf84d1e62c1fd1a4c48407360afea9494a387eefb74627b46000210a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"77f4bb331d7c87c8c6f645d4b5f13f08407295d66ee31371666f5da75544d885","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x371874876980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ftnkcdsss4wg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\",\n \"name\": \"883d0d0d-c572-41f2-9f82-7d61a050db28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:27:29.613460957Z\",\n \"endTime\": \"2026-10-08T14:56:34.939890509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x371874876980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ftnkcdsss4wg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\",\n \"name\": \"883d0d0d-c572-41f2-9f82-7d61a050db28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:27:29.613460957Z\",\n \"endTime\": \"2026-10-08T14:56:34.939890509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7f918c88fe9ad08671ad61d6e542244ee013a07babccaa8d5c4c7988234f7476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"04197c3f7c272281d2f16a127440b51aecd5581858f715b36eb52e6c706ac7dc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x6335433a090\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x6335433a090\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"808fe86b6fa37a637289bcc47cc0556cffbf7f45c8d4d6e766be90d644a32592","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"cea1f06c65c9ac1a5240a3f2299004282ac5955daa892019aaf1e458ea109d8a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ac3b36042c0\u003e: \n failed to create NodePool np-ctg8q4: failed waiting for nodepool=\"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\" in resourcegroup=\"rg-candidate-4-23-z87lsqfql56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n \"name\": \"4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:57:55.181740971Z\",\n \"endTime\": \"2026-10-08T15:27:00.08772636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ac3b36042c0\u003e: \n failed to create NodePool np-ctg8q4: failed waiting for nodepool=\"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\" in resourcegroup=\"rg-candidate-4-23-z87lsqfql56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n \"name\": \"4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:57:55.181740971Z\",\n \"endTime\": \"2026-10-08T15:27:00.08772636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8af9fc1debd8997228c5dc50c63c36c5235c97b4afe1a21148fad7a3d230225d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"6771b69c9cd8b6f46bf94cd51d146bf6cecc95e92b532d2cb269b24721b30ded","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"8b0c89e21557e2d717f89f8dbae5f26d924f555e9393c99fbd8b4ca42f7d7934","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"5f3bfb947330e190172ba5110d4435e2bf0388db1bc378c5cd2b5067c49ec984","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36b6bf7fec00\u003e: \n failed to create NodePool np-24nxrk: failed waiting for nodepool=\"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\" in resourcegroup=\"rg-candidate-5-1-74n6vxnsbzln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n \"name\": \"a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:58:11.889421655Z\",\n \"endTime\": \"2026-10-08T15:27:20.151153667Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36b6bf7fec00\u003e: \n failed to create NodePool np-24nxrk: failed waiting for nodepool=\"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\" in resourcegroup=\"rg-candidate-5-1-74n6vxnsbzln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n \"name\": \"a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:58:11.889421655Z\",\n \"endTime\": \"2026-10-08T15:27:20.151153667Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a9083cffcb27d3e238ee2b45350b0033e7e9c2b5ce19ab861e517d7ce6b8bc52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"78b9b25829be4c3984fe03924f28bc2a19d7a1e5fbcf3047833b3465f0d57419","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ad692bfa2c0\u003e: \n failed to create NodePool np-fgmnm8: failed waiting for nodepool=\"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\" in resourcegroup=\"rg-candidate-4-22-t7czqbb9gdw6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n \"name\": \"df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:00:21.098278607Z\",\n \"endTime\": \"2026-10-08T15:29:30.220139004Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ad692bfa2c0\u003e: \n failed to create NodePool np-fgmnm8: failed waiting for nodepool=\"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\" in resourcegroup=\"rg-candidate-4-22-t7czqbb9gdw6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n \"name\": \"df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:00:21.098278607Z\",\n \"endTime\": \"2026-10-08T15:29:30.220139004Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b9c1d59bebd237518de05195e7ca383e1f832d136cdaf30d63742c9777710ec4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"d7c2cc02f3a607ef7b6cbe3b49264471171e7681498d80954c9331231b7a8a89","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x248495222160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2s5lcfmgt854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\",\n \"name\": \"93a34e6e-477c-4e65-9e08-527e17028921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:23:16.915326226Z\",\n \"endTime\": \"2026-10-08T14:52:17.249683042Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-xxpw5-lhcfb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-xxpw5-lhcfb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x248495222160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2s5lcfmgt854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\",\n \"name\": \"93a34e6e-477c-4e65-9e08-527e17028921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:23:16.915326226Z\",\n \"endTime\": \"2026-10-08T14:52:17.249683042Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-xxpw5-lhcfb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-xxpw5-lhcfb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c15ae007646bc762cd892187b7b2b65201cecb0ceabb05c268d2087fb8625d51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"ccc88a84bcfc47cb169e8a095ef78886581bb9c45ae50908d9d1797d28040551","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c37b4b9e200\u003e: \n failed to create NodePool np-fsgggm: failed waiting for nodepool=\"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\" in resourcegroup=\"rg-candidate-4-20-rz65zsts87v9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\",\n \"name\": \"240398ab-2ae4-4b7b-a355-04e9e2139068\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:57:34.48901399Z\",\n \"endTime\": \"2026-10-08T15:26:35.280704579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c37b4b9e200\u003e: \n failed to create NodePool np-fsgggm: failed waiting for nodepool=\"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\" in resourcegroup=\"rg-candidate-4-20-rz65zsts87v9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\",\n \"name\": \"240398ab-2ae4-4b7b-a355-04e9e2139068\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:57:34.48901399Z\",\n \"endTime\": \"2026-10-08T15:26:35.280704579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf1eff9fbf03650fdced227e66686ce898bc70324b002192c474fe40982ab577","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"524a2a76e90c2d1da53a5c0a9582d861268831f07109dc1e7db02e6ed4141945","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2a3a9b00e6d8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hxkght4ww89l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n \"name\": \"f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:28:38.920968051Z\",\n \"endTime\": \"2026-10-08T14:57:44.886200601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2a3a9b00e6d8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hxkght4ww89l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n \"name\": \"f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:28:38.920968051Z\",\n \"endTime\": \"2026-10-08T14:57:44.886200601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e340178c0d8e0d0d1dc84cbc7a9fd1bd10bf04551585781678969d821f003b1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"ca182933bc653faf7eb30127aebd6ed595c5e97059da875779383f3b49535773","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xa91b8f312a0\u003e: \n failed to create NodePool np-c7f6vs: failed waiting for nodepool=\"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\" in resourcegroup=\"rg-candidate-5-0-mlpng9ckwtsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\",\n \"name\": \"630087f9-3e91-4cc3-95cc-3a0875463d79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:56:46.681105188Z\",\n \"endTime\": \"2026-10-08T15:25:55.347852762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xa91b8f312a0\u003e: \n failed to create NodePool np-c7f6vs: failed waiting for nodepool=\"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\" in resourcegroup=\"rg-candidate-5-0-mlpng9ckwtsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\",\n \"name\": \"630087f9-3e91-4cc3-95cc-3a0875463d79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:56:46.681105188Z\",\n \"endTime\": \"2026-10-08T15:25:55.347852762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T03:46:27Z","started_at":"2026-10-08T03:46:27Z","completed_at":"2026-10-08T06:41:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T06:58:48.848958423Z","timing_metadata_checked_at":"2026-10-08T06:58:48.848958423Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T06:58:48.853270609Z","region_metadata_checked_at":"2026-10-08T06:58:48.853270609Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"0689ca5a0065e3d8823a728436aa2c2d412e543792f8a8e3f6fb0c02d473d41e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"c07cc1511d492c4af5906301b6ac8729ea5b2acd7f223c5094e456cf7e259b7e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19e2b2505020\u003e: \n failed to create NodePool np-cmn25n: failed waiting for nodepool=\"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\" in resourcegroup=\"rg-candidate-5-0-lhc5fj69t9pm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\",\n \"name\": \"6879bb7c-c254-4641-ba70-7b41bb04b080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:52:10.585592043Z\",\n \"endTime\": \"2026-10-08T05:21:19.889088931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19e2b2505020\u003e: \n failed to create NodePool np-cmn25n: failed waiting for nodepool=\"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\" in resourcegroup=\"rg-candidate-5-0-lhc5fj69t9pm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\",\n \"name\": \"6879bb7c-c254-4641-ba70-7b41bb04b080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:52:10.585592043Z\",\n \"endTime\": \"2026-10-08T05:21:19.889088931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"321d9f3948c5613ff9f1f989e13aac53c1d34d5d42d91a0b942f14c62a289b86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"0e5d065bab30d941a1416295359f6c50ec9d353a91602de79cd9253e6bac9e86","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1f97b95fe138\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zl88ncfz4jp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n \"name\": \"4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:18:50.555017189Z\",\n \"endTime\": \"2026-10-08T04:47:59.745762657Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1f97b95fe138\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zl88ncfz4jp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n \"name\": \"4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:18:50.555017189Z\",\n \"endTime\": \"2026-10-08T04:47:59.745762657Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3656a44f641fb97a49d342f9557cfc341900c599d57160568ab694fe9bfd652d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"ca5e81422055ee4bd8d2f8df0fa691b5dc11cb7cece132e6083770e89a70662a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x228c1da4a2e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-tff6vqrfwntq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\",\n \"name\": \"40cea923-3d28-46b1-b377-958e6c7d32be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:17:53.289622119Z\",\n \"endTime\": \"2026-10-08T04:46:59.632942955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-7vfqr-gvgx9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-7vfqr-gvgx9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x228c1da4a2e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-tff6vqrfwntq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\",\n \"name\": \"40cea923-3d28-46b1-b377-958e6c7d32be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:17:53.289622119Z\",\n \"endTime\": \"2026-10-08T04:46:59.632942955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-7vfqr-gvgx9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-7vfqr-gvgx9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"38a0864e490ba06214861a9d18cb9804384f31e1146a2bf35df81690d2dbd4ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"212d4cea3adecaa7fb0b5b48ed66e7c9510a248ab4a2550d0e77537380105fa1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xbf2bf37c078\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rgbgvj2hmvgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n \"name\": \"5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:21:09.226625204Z\",\n \"endTime\": \"2026-10-08T04:50:09.941476385Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xbf2bf37c078\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rgbgvj2hmvgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n \"name\": \"5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:21:09.226625204Z\",\n \"endTime\": \"2026-10-08T04:50:09.941476385Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"489dfac5fc01d6e197f333a0cb9254116fdefe570bed484df3321871d4b07956","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"f6915c2acdb65b50b5f491323caf68c80329ffbf00cf123f661b2d3be53776ae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28a272d90ba0\u003e: \n failed to create NodePool np-2wf5g9: failed waiting for nodepool=\"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\" in resourcegroup=\"rg-candidate-5-1-tpzmdkt924rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n \"name\": \"acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:52:41.296917226Z\",\n \"endTime\": \"2026-10-08T05:21:49.770952812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28a272d90ba0\u003e: \n failed to create NodePool np-2wf5g9: failed waiting for nodepool=\"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\" in resourcegroup=\"rg-candidate-5-1-tpzmdkt924rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n \"name\": \"acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:52:41.296917226Z\",\n \"endTime\": \"2026-10-08T05:21:49.770952812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"530cb9faba5bf5132137f74587df0222fb8c053cb05cf4db6c02b63255eefbac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"50a678b577dd6ec95a39c4930784d2f2f8df4c2e4deb6c817d27050680bc7bad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1acc4afec200\u003e: \n failed to create NodePool np-5fp562: failed waiting for nodepool=\"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\" in resourcegroup=\"rg-candidate-4-22-x7bk4qfg72wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n \"name\": \"df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:49:21.745083143Z\",\n \"endTime\": \"2026-10-08T05:18:29.866533862Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1acc4afec200\u003e: \n failed to create NodePool np-5fp562: failed waiting for nodepool=\"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\" in resourcegroup=\"rg-candidate-4-22-x7bk4qfg72wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n \"name\": \"df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:49:21.745083143Z\",\n \"endTime\": \"2026-10-08T05:18:29.866533862Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"868418b3541d83837afb2bf28b3129a904a615b0c35d6a0ff4f986bc1db7f86b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"e31120375fe20c3189a3f2c035d3f9c8bac3cf1283b6c17aad50ebf324b212ad","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"875996f784515e3801c3cd8bba5867d64e55f47b9f8805078f0000b6cceabba6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"8d541c9f52f89327eb524d55f2cbc5184388fedfbf5f16fa8cb7e24ae40f7f99","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cad2d10c0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xktww2hxb7t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n \"name\": \"7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:19:51.649780345Z\",\n \"endTime\": \"2026-10-08T04:48:59.779189852Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cad2d10c0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xktww2hxb7t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n \"name\": \"7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:19:51.649780345Z\",\n \"endTime\": \"2026-10-08T04:48:59.779189852Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c7309cec05c5118a8a3cdc8dcc321027b2e593cf1616511ea3783d0cf55bf5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"bc2d06f2e3bbb11b62e2bdbb0371b19a15817c0acbbe41a8a602481250c62016","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e5feac2cf20\u003e: \n failed to create NodePool np-nm4ft4: failed waiting for nodepool=\"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\" in resourcegroup=\"rg-candidate-4-20-gwp4bdzvdkfn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\",\n \"name\": \"7d0b1c33-9150-4065-8244-327119876e65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:47:03.009170999Z\",\n \"endTime\": \"2026-10-08T05:16:09.831739591Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e5feac2cf20\u003e: \n failed to create NodePool np-nm4ft4: failed waiting for nodepool=\"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\" in resourcegroup=\"rg-candidate-4-20-gwp4bdzvdkfn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\",\n \"name\": \"7d0b1c33-9150-4065-8244-327119876e65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:47:03.009170999Z\",\n \"endTime\": \"2026-10-08T05:16:09.831739591Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b087af5f901e9c6fcea71002963c03e8728c722c9dce6b700d1fe4f3d21e8806","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"704461f11fba58d3264f2375448ec8d0dcfefed9be4b8d25b1805dc09d5f9d85","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27ed88bbf980\u003e: \n failed to create NodePool np-kjsxrn: failed waiting for nodepool=\"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\" in resourcegroup=\"rg-candidate-4-23-9blkbrrd87j2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\",\n \"name\": \"ea16240d-6235-4ea8-8e63-3873031d6d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:54:34.518182875Z\",\n \"endTime\": \"2026-10-08T05:23:35.635646755Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27ed88bbf980\u003e: \n failed to create NodePool np-kjsxrn: failed waiting for nodepool=\"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\" in resourcegroup=\"rg-candidate-4-23-9blkbrrd87j2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\",\n \"name\": \"ea16240d-6235-4ea8-8e63-3873031d6d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:54:34.518182875Z\",\n \"endTime\": \"2026-10-08T05:23:35.635646755Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c4d2d9d1b56975646ec96bf8c7f196d445714641cd5788d5adae0109a6f8033c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"b9836bfff02572891050840308d6c8bb9be4b2b018e321a4135572ee218da36e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39a1a2b36a20\u003e: \n failed to create NodePool np-5bpm5c: failed waiting for nodepool=\"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\" in resourcegroup=\"rg-candidate-4-21-vptzpz86kwlt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\",\n \"name\": \"5b709679-17d4-47e9-a453-c0f7086a9226\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:50:41.026700325Z\",\n \"endTime\": \"2026-10-08T05:19:49.834642675Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x39a1a2b36a20\u003e: \n failed to create NodePool np-5bpm5c: failed waiting for nodepool=\"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\" in resourcegroup=\"rg-candidate-4-21-vptzpz86kwlt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\",\n \"name\": \"5b709679-17d4-47e9-a453-c0f7086a9226\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:50:41.026700325Z\",\n \"endTime\": \"2026-10-08T05:19:49.834642675Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:00:37Z","started_at":"2026-10-08T02:00:37Z","completed_at":"2026-10-08T04:24:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T04:39:48.472775399Z","timing_metadata_checked_at":"2026-10-08T04:39:48.472775399Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T04:39:48.476543988Z","region_metadata_checked_at":"2026-10-08T04:39:48.476543988Z"},"failed_at":["e2e"],"failed_test_count":34,"occurrences":[{"row_id":"14966e10511ef08503c658c34d2b5114bc8c570733e90d6d7188aa635b328b7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"11fb9973540ad640cafcdd9a78df56a498b2cd9d499c2536d6c8e235215b9784","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bcb55dd85a0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-crphwg\" resourcegroup=\"rg-np-dg-minor-crphwg-6m9n6b7fcg8d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-crphwg-6m9n6b7fcg8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-crphwg/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bcb55dd85a0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-crphwg\" resourcegroup=\"rg-np-dg-minor-crphwg-6m9n6b7fcg8d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-crphwg-6m9n6b7fcg8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-crphwg/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"186408d744410d84655ae04ddeda71286bbf193d5137fa4d86aec803863bc6bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"7bc1860acf1bdc160e1d9d3e083e6a77b4a25446ec861b06bce73c5aef74c5d0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x23e3b649efa0\u003e: \n failed to create NodePool np-8tzhmt: failed waiting for nodepool=\"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\" in resourcegroup=\"rg-candidate-4-22-sbbrjn549jd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\",\n \"name\": \"33868d99-b424-4bf0-8b0a-debdff616e75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:47:01.121478067Z\",\n \"endTime\": \"2026-10-08T03:16:05.005157282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x23e3b649efa0\u003e: \n failed to create NodePool np-8tzhmt: failed waiting for nodepool=\"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\" in resourcegroup=\"rg-candidate-4-22-sbbrjn549jd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\",\n \"name\": \"33868d99-b424-4bf0-8b0a-debdff616e75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:47:01.121478067Z\",\n \"endTime\": \"2026-10-08T03:16:05.005157282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1bd558ac86b721d816be67679fb2a0e075e7fa69155d9a9d03fb6195f33ccb05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"dc21971b0a431404b75ca2aa3a8a2071c2727c9e220c888a3e332140a7c42a55","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1052ae90d2e0\u003e: \n failed to create NodePool np-wlzmmg: failed waiting for nodepool=\"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\" in resourcegroup=\"rg-candidate-5-1-2rb9r54kz678\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\",\n \"name\": \"ba8e191c-888b-485b-8866-d49e0334cc7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:51:25.134077752Z\",\n \"endTime\": \"2026-10-08T03:20:27.169387826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1052ae90d2e0\u003e: \n failed to create NodePool np-wlzmmg: failed waiting for nodepool=\"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\" in resourcegroup=\"rg-candidate-5-1-2rb9r54kz678\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\",\n \"name\": \"ba8e191c-888b-485b-8866-d49e0334cc7c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:51:25.134077752Z\",\n \"endTime\": \"2026-10-08T03:20:27.169387826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2177f95f365731cbb0ea068d0849d2e35cbd9f556ca72e6613f3559b49b28e40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"6e99dc03e140d021122b5f99a0b39eb90363b2a3eb418fc13a8bcf957465177e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x4fa9c508420\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s9lxbp6g9wp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x4fa9c508420\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s9lxbp6g9wp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"27a2e14421f895218bfd5bb145d8093811f0acb482eac4dd6c9c185e50d85a77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"9f77071dc6747b0314cd8920ca2287311b939a55c7de09fa54be0bfe2bcd1f78","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"2b77fd07245ac0d996ddbda292b0b13fde4308ef75ba39d22a347310328fd073","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"d9ecb82f70950229f902353084a6415477e147720c50fc7d251959e1cc792350","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-l5lgrv\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c5666c0fce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-hhxjcw4kwjpv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-l5lgrv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-l5lgrv\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c5666c0fce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-hhxjcw4kwjpv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-l5lgrv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2c3ec25b0d3f13a7fe9a1535fc096e04c42ca1919d375441490495bb85f80743","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"81166f4884fdcd117896960c8c39ef279bd4c30f7dfcfe06334faafd896c846c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-bd76nk\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1ad51ba8c390\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-xg2dpqfdj5bt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bd76nk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-bd76nk\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1ad51ba8c390\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-xg2dpqfdj5bt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bd76nk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"320e9ebe656e8a7cf72010184d0ace936716aa64d4a479519c6113d40fca1a21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3979048c452a40f54f92711abf6015db7d4a3247bf1fce0134b9a35f9c83290f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-kdl52j\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xed02447ca80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-lljlhvtp7gqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-kdl52j\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-kdl52j\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xed02447ca80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-lljlhvtp7gqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-kdl52j\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"38afadc8f4353feef22f22bb58447a449c3516a7ac6e0105e16c5fd206646d31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"8dbd91ef71ec969c7b63355f72e7f62a344f89bc68b893cbfcc89d6c2e547cb5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb57001bd580\u003e: \n failed to create NodePool np-tkcz74: failed waiting for nodepool=\"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\" in resourcegroup=\"rg-candidate-4-21-qcxhk95mnhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\",\n \"name\": \"95120e83-d81e-45c6-bda2-21b985646077\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:45:34.00484297Z\",\n \"endTime\": \"2026-10-08T03:14:34.986063906Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb57001bd580\u003e: \n failed to create NodePool np-tkcz74: failed waiting for nodepool=\"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\" in resourcegroup=\"rg-candidate-4-21-qcxhk95mnhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\",\n \"name\": \"95120e83-d81e-45c6-bda2-21b985646077\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:45:34.00484297Z\",\n \"endTime\": \"2026-10-08T03:14:34.986063906Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"39e49c472e534edf65f97e338eb433fbffcaa6834b0a46d75ea8454b7de89eb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3d17f8d6652d176f544f7ddbab6bf98de82badadb74a026139b33bc1b25c0814","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x6b03fb724a0\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-4bzrjj\" resourcegroup=\"rg-np-noedge-4bzrjj-n97rz8gmwdvr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4bzrjj-n97rz8gmwdvr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-4bzrjj/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x6b03fb724a0\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-4bzrjj\" resourcegroup=\"rg-np-noedge-4bzrjj-n97rz8gmwdvr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4bzrjj-n97rz8gmwdvr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-4bzrjj/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"406c3dac759539db26f3e6fbecfb151a13c1309138a71e07be0da3341169f845","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"93c5f443b65f365a99e52dd59b3e31b6da3855ac65193c7fc2b456d85c0da930","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0xc813599cf60\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-dz427r\" resourcegroup=\"rg-np-skip-minor-dz427r-crp6tpsxwgq6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dz427r-crp6tpsxwgq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dz427r/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0xc813599cf60\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-dz427r\" resourcegroup=\"rg-np-skip-minor-dz427r-crp6tpsxwgq6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dz427r-crp6tpsxwgq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dz427r/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"47502119e4a155db0b63d38c1ce3e9a1ce2bb015a9e8b8efdcbebc1ed3de3df2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"35790e776fa5175e4de78d7f6617ad98c96a8f158a3232735e601d4e5b778256","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-64zx6d\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xd58a144ed0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-dstvpqcxjmxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-64zx6d\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-64zx6d\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xd58a144ed0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-dstvpqcxjmxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-64zx6d\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4f1b777d68d2307c0345d55a91d231746a41a42a9d2fa63504fdca50d2154ef0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"399ce0931f87760834405b1f91f2bb0dc5c6dee722c1e4c00e390a98ef2e93ec","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2c220eff60c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bxcw427xldqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n \"name\": \"f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:18:39.603761603Z\",\n \"endTime\": \"2026-10-08T02:47:44.845130459Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2c220eff60c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bxcw427xldqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n \"name\": \"f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:18:39.603761603Z\",\n \"endTime\": \"2026-10-08T02:47:44.845130459Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"540fe004db59d57632a6eb31c42efbb741c42918fba74f33543adce57ab98798","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"f6fc53eb549f0e7f2f95375c5420737172608e9674edf0be196e6c524d2b9c9a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0x26e1bcba84a0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-7jfzmq\" resourcegroup=\"rg-np-downgrade-7jfzmq-rvr22sztdn98\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-7jfzmq-rvr22sztdn98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-7jfzmq/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0x26e1bcba84a0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-7jfzmq\" resourcegroup=\"rg-np-downgrade-7jfzmq-rvr22sztdn98\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-7jfzmq-rvr22sztdn98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-7jfzmq/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"55307805d8a8b30cfdbf4a4b595402a521d427fa95b511f792a626f713c1eddd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"1609838bf5872d5e2d583f56af041bc1d204c93d24ccdb32dad101d6536f4ebf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-hn2cnv\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x9116922ea20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-4w8m65kzp5nn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-hn2cnv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-hn2cnv\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x9116922ea20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-4w8m65kzp5nn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-hn2cnv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"699b0f65b5a1824d30c05839809075ac3099a2b3c1b8eeb56be0e7632810fc07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"f0bf9d5656cd4403fe7ec557e934ab25ebc30cdf02bf25bdc5454a196c5801bd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-xjwrx8\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f5c96b45080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lt88zk4qvv8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-xjwrx8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-xjwrx8\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f5c96b45080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lt88zk4qvv8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-xjwrx8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6b804eed762d5abeb2a719c1e2de9ff084e9104c02e3bb135ccd25901a849f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"ddf416b60b13f8a7895320145ea311edeee5e97b0e336d70a0126bdfa4e1c5e4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x28f92e3a99c0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-tw9ncc\" resourcegroup=\"rg-np-version-upgrade-tw9ncc-98zjd59jkjpr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tw9ncc-98zjd59jkjpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tw9ncc/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x28f92e3a99c0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-tw9ncc\" resourcegroup=\"rg-np-version-upgrade-tw9ncc-98zjd59jkjpr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tw9ncc-98zjd59jkjpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tw9ncc/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6d1cb8c8f743985adb1b2f2947b9f876135b55a694f240adc8d8bb997127ef4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"d9b09315f5cbffbd9b174578ef412adca169d6ebcfe103e2e14792ace908df4b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x65e48efd800\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-qwp7bbpclw7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n \"name\": \"1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:19:11.90853804Z\",\n \"endTime\": \"2026-10-08T02:48:14.711261391Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x65e48efd800\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-qwp7bbpclw7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n \"name\": \"1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:19:11.90853804Z\",\n \"endTime\": \"2026-10-08T02:48:14.711261391Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"786928115e7b85b965612e45ed5d08b9e7c17fb5987ee1cf25a3528656e6456f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3432fe66c4ed90923c24f3ae7f15c3a4bfc33f20470b8a1b2b57487830b17e6b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-x2nmgc\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x3577745b47e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l49w7x6sf8lk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x2nmgc\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-x2nmgc\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x3577745b47e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l49w7x6sf8lk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x2nmgc\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8a29209d4da02b2a146bed074ed7c21a78ba34f0531b211eee146507a6ec9548","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"697bacd58389e5a2c004217397ccee804ff03a55b0df4328e472ad3b6538aa9d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x35b2e2ff91e0\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-rx2rkxms8sw7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-rx2rkxms8sw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x35b2e2ff91e0\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-rx2rkxms8sw7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-rx2rkxms8sw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"902867b8d922a670ddb97e92b74160c752642686b8f29b0615242c7618b7b694","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"4ea5e665e1441c36ffd5c3ff6c454b7210f8c32ad40846122aa42883cbc59638","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wps54w\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ddfbef4d200\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-9qcq2bc8pfhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-wps54w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wps54w\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ddfbef4d200\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-9qcq2bc8pfhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-wps54w\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"91294f3642fd3e7467f1226eb5fec14c8a86fb66ed7d8476319a11ffb18aad77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"286f2b337db1c06afe4f552fafe360d2df35c905db51dc5712e0039ace5c0cf3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17bbc4694220\u003e: \n failed to create NodePool np-znb5qj: failed waiting for nodepool=\"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\" in resourcegroup=\"rg-candidate-5-0-d5trjc74qhsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\",\n \"name\": \"14de4838-40ca-415c-a29b-7d78f06ecdac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:49:59.670336775Z\",\n \"endTime\": \"2026-10-08T03:19:07.230773535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17bbc4694220\u003e: \n failed to create NodePool np-znb5qj: failed waiting for nodepool=\"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\" in resourcegroup=\"rg-candidate-5-0-d5trjc74qhsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\",\n \"name\": \"14de4838-40ca-415c-a29b-7d78f06ecdac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:49:59.670336775Z\",\n \"endTime\": \"2026-10-08T03:19:07.230773535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"91a8fb037681201f0071057c78e2f37fc7a1549554fe857363d49c520cbc690c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"71ce598d6d9d958fde2cd619eb779c67cc6e323a3cce0328ebb3cc5eff50003c","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"NotFound\\\",\\r\\n \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8da18459eef6017d12822e9dd3b87c09fdb695ab8d9cb4881da243c125e2db6b","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"926d351375051e89ff778a8ba9a0efe4eac5609a965e70c5e6db97f7c3254f9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"49bc5c5cdc05073630918f616f1b204bce3a50fc2a76296971ab18ebc0822591","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x6f28c1d0c00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vpk5q2z5574v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x6f28c1d0c00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vpk5q2z5574v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"adf4d43683d33e78991e33268ad452577c023245ad376952ed4efbb985f06ff8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"e64509f96f2f07ea596d1a6b68e536a246bfe0c47479f375bb73c60def84644c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x137051db4300\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hcx5lwfp6892/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x137051db4300\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hcx5lwfp6892/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b2ad1169a1cb2a22fcf63f820b28f514d60dec0e7b7884b5c76269a16b9c8f7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"97e9cfb4d9ae6fd232412678c19b7ea57ae3f6d1c4164518dd3f623a9673d2b5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d2ef35d8e0\u003e: \n failed to create NodePool np-xhn5bs: failed waiting for nodepool=\"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\" in resourcegroup=\"rg-candidate-4-23-4llsqxjhkgt4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\",\n \"name\": \"93da1929-fde9-4a74-85a6-ad361a679f40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:49:02.43694261Z\",\n \"endTime\": \"2026-10-08T03:18:07.302711356Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d2ef35d8e0\u003e: \n failed to create NodePool np-xhn5bs: failed waiting for nodepool=\"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\" in resourcegroup=\"rg-candidate-4-23-4llsqxjhkgt4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\",\n \"name\": \"93da1929-fde9-4a74-85a6-ad361a679f40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:49:02.43694261Z\",\n \"endTime\": \"2026-10-08T03:18:07.302711356Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b7fdc8faa0cae5129368e672d232ca363da3b30023302654984e53151b773744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"a502c3c33e95830c6b055e8b63c99d465d434e71cd54d550ac664ae0376a0a52","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2edd5c272b00\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-df62qq\" resourcegroup=\"rg-np-version-upgrade-df62qq-l9cc8d44b2p9\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-df62qq-l9cc8d44b2p9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-df62qq/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2edd5c272b00\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-df62qq\" resourcegroup=\"rg-np-version-upgrade-df62qq-l9cc8d44b2p9\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-df62qq-l9cc8d44b2p9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-df62qq/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b8b1332ab3598b4e81c31f5b81723ee5d77d00be4dfcf92b9e2f8f5408302135","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"8eb81065221254a41909e446e130e2c5564d04ecbae268b12a83dd5adbb708e4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b63d32ea960\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-6l2qnmrqmq6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\",\n \"name\": \"987e3697-e775-428f-b96b-007cf98cd200\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:17:06.289611141Z\",\n \"endTime\": \"2026-10-08T02:46:14.822515799Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-b4b48-6xc2t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-b4b48-6xc2t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b63d32ea960\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-6l2qnmrqmq6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\",\n \"name\": \"987e3697-e775-428f-b96b-007cf98cd200\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:17:06.289611141Z\",\n \"endTime\": \"2026-10-08T02:46:14.822515799Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-b4b48-6xc2t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-b4b48-6xc2t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c141a316ba7b115d1fd98d2a343639c8170a433641ad2914b822a5d85e840df5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"66b6c637f092b0a1ae34328f0c47a9d939259c22ca517a9d0e2276a87a3b13b0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-xf8crr\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x28f031e29260\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-mqfd4qzn4578/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-xf8crr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-xf8crr\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x28f031e29260\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-mqfd4qzn4578/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-xf8crr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c272f364b73991a037f7d63fa1c19bfbcd79db8802deff82f0ac36a869071f33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"363bec732de5d54b45f82f6c6410eef0cdb5ff0af00c3fc2788bcb78b1e622e4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe292679e030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwdf5gjhw97g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\",\n \"name\": \"e74aa6d0-c8cd-430c-8190-549622080499\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:18:33.237818266Z\",\n \"endTime\": \"2026-10-08T02:47:34.745399444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe292679e030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwdf5gjhw97g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\",\n \"name\": \"e74aa6d0-c8cd-430c-8190-549622080499\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:18:33.237818266Z\",\n \"endTime\": \"2026-10-08T02:47:34.745399444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c286511b3464afb3bafc9a8e0ba470fb745b7cdd6e8c608c87d498f3d7dccfb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"e088dc0de9a1fac4cac481a667a95c6c5ae11cc93acca3216cdc27fe06be74e3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-pcrbcd\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x33c001cb49c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zb4tmg42vgq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-pcrbcd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-pcrbcd\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x33c001cb49c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zb4tmg42vgq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-pcrbcd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c95790ba8f7de78ac9b61dbb2e0d12efd08af034e14ee49a49d3889434fdb858","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"b8dc64cff9c94bedff3d6968dd7a64f15705facf6a41874e487dabb240ea1551","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x19c5b50bc860\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-x2784j5xhvrh\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-x2784j5xhvrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x19c5b50bc860\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-x2784j5xhvrh\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-x2784j5xhvrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cda5197cc3247bba4a9f43335d43a6a3400bef72be79e52d7582b5dcc6b97dc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"a2308670e8d0cf661e1d2de3f880177522e71068292186eb5069f5861f9c634c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x130c0be607c0\u003e: \n failed to create NodePool np-gm6fx9: failed waiting for nodepool=\"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\" in resourcegroup=\"rg-candidate-4-20-ts84fqwf8d5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\",\n \"name\": \"b9435cae-155f-47ef-8d90-e886b71aff15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:45:36.63094836Z\",\n \"endTime\": \"2026-10-08T03:14:44.806443674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x130c0be607c0\u003e: \n failed to create NodePool np-gm6fx9: failed waiting for nodepool=\"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\" in resourcegroup=\"rg-candidate-4-20-ts84fqwf8d5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\",\n \"name\": \"b9435cae-155f-47ef-8d90-e886b71aff15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:45:36.63094836Z\",\n \"endTime\": \"2026-10-08T03:14:44.806443674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f2deb49c8a8aca27af34ff3c7804bda6c0614ebda96975b4e4318262681ce1ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"759bd9f85e38e37d1860e839ceb34b5c137190cb65b706eb8146bc3e9eb17622","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xa63e00795e0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-dwrcvf\" resourcegroup=\"rg-np-version-upgrade-dwrcvf-tndq67k8xt89\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dwrcvf-tndq67k8xt89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-dwrcvf/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xa63e00795e0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-dwrcvf\" resourcegroup=\"rg-np-version-upgrade-dwrcvf-tndq67k8xt89\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dwrcvf-tndq67k8xt89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-dwrcvf/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":34,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","8da18459eef6017d12822e9dd3b87c09fdb695ab8d9cb4881da243c125e2db6b","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":34,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T23:00:38Z","started_at":"2026-10-08T23:00:38Z","completed_at":"2026-10-09T00:33:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:49:25.67324716Z","timing_metadata_checked_at":"2026-10-09T00:49:25.67324716Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:49:25.67672975Z","region_metadata_checked_at":"2026-10-09T00:49:25.67672975Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"2b2f71c4c571919297984db3c722cbabc74c18e6ec0303a2f98f387824e5587e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"6a786858a0ce9d8898d3bee0c6e0bb2b32e2b8290a00cd0d692d1083e2a1c7db","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3c35adc80030\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-clb6t6k9tfdf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3c35adc80030\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-clb6t6k9tfdf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"64720a21fb23e09795e227cef47abfb9e3977fedd9656e8867c3957737dd28ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"9d68ac0cb994edda68255e9dddbaf9ce4fa7933fd1c252de20b095bdb9081302","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-20-g5bgdz\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x2608a89861e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2608a90941e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2608a90941c0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2608a81192c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2608a86f4180\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2608a86f4100\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2608a7f286e0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-20-g5bgdz\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x2608a89861e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2608a90941e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2608a90941c0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2608a81192c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2608a86f4180\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2608a86f4100\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2608a7f286e0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eba9f0207784082ae028e0292575147a8480a2f1068a534a4622234669f79e31","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"99524a5ee5638b52807bdc7e80460171fc7ba7b40d72473d1c83b37592a70b0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"b206c5ae2b103cbfe005db15a1c02fa20fa420980cde7e1618e434e35e0da46e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x226dedcde000\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-c4d5nkzpw5nz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x226dedcde000\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-c4d5nkzpw5nz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9f4e2b3046ecbe062c9586a0d3d5595427aa870776a80c089c1e1b86c278b948","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"4755271a2405ebabdc1040cbbd3db7c2ac9758569fd67a3df2b4609761755cde","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x18f7dc4001e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-v2s5z7rdtt9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x18f7dc4001e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-v2s5z7rdtt9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e1fc71c1c006ade3eeb35bd68d7a633af4e1472a223bf2c5a97fee8bb4b59230","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"f88d47a77e4c59b900e74f056ed5c5b9d70d2185cfe256eba9c161d0c2e7ae93","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2add6a662db0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-wkvrqn6clxhk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2add6a662db0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-wkvrqn6clxhk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fcc3d49a0e2dc589fba80b5edb9baadf2be8096d47ad9674ab87e7326ef14fe0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"d6925f99dc273a7ab082759d35af63245cb77400725a1a98efeefc4b973a6b03","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x284f14134240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sf6285dvgkzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x284f14134240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sf6285dvgkzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","eba9f0207784082ae028e0292575147a8480a2f1068a534a4622234669f79e31"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2108272446057484288","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T19:04:55Z","started_at":"2026-10-08T19:04:55Z","completed_at":"2026-10-08T21:29:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:42:16.323843169Z","timing_metadata_checked_at":"2026-10-08T21:42:16.323843169Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T21:42:16.332322943Z","region_metadata_checked_at":"2026-10-08T21:42:16.332322943Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2108014673335947264","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:00:37Z","started_at":"2026-10-08T02:00:37Z","completed_at":"2026-10-08T03:02:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:16:09.739745848Z","timing_metadata_checked_at":"2026-10-08T03:16:09.739745848Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-08T03:16:09.743748735Z","region_metadata_checked_at":"2026-10-08T04:19:36.935397323Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6ca54a97226f02c5e8b9117e6709f8588f56a1d01de3efb70036b1b3d8b627b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2108014673335947264","occurred_at":"2026-10-08T02:00:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T00:03:34Z","started_at":"2026-10-08T00:03:34Z","completed_at":"2026-10-08T03:13:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T03:26:12.94208938Z","timing_metadata_checked_at":"2026-10-08T03:26:12.94208938Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T03:26:12.946500565Z","region_metadata_checked_at":"2026-10-08T03:26:12.946500565Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"05fb17bfc6837f9ac7209d82a4c2a5ae77675c293314d714712444cdf075e78a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"9cbc0a353d999f7daabf373bfd9ff5feac8ee9d23b5b5e47d914749b0e6d8820","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1480002e0090\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v6mhpvmjllbc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f4505ea-893f-413a-81c7-aa357a711eaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f4505ea-893f-413a-81c7-aa357a711eaa\",\n \"name\": \"9f4505ea-893f-413a-81c7-aa357a711eaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:19:19.539417138Z\",\n \"endTime\": \"2026-10-08T01:48:27.529510506Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-npz5p-fm9t2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-fm9t2 to exist\\nMachine cilium-cluster-cilium-np-npz5p-pln66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-pln66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-npz5p-fm9t2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-fm9t2 to exist\\nMachine cilium-cluster-cilium-np-npz5p-pln66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-pln66 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1480002e0090\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v6mhpvmjllbc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f4505ea-893f-413a-81c7-aa357a711eaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f4505ea-893f-413a-81c7-aa357a711eaa\",\n \"name\": \"9f4505ea-893f-413a-81c7-aa357a711eaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:19:19.539417138Z\",\n \"endTime\": \"2026-10-08T01:48:27.529510506Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-npz5p-fm9t2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-fm9t2 to exist\\nMachine cilium-cluster-cilium-np-npz5p-pln66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-pln66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-npz5p-fm9t2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-fm9t2 to exist\\nMachine cilium-cluster-cilium-np-npz5p-pln66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v6mhpvmjllbc--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-npz5p-pln66 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"199a35f91894c330bded5c02b48db2fe4b934c8f199f44c1be44ed46f855c3ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"31263bdaf37c31d8b00994a69910ef2e640144e0087ea9bca2ce2644230f566e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vdrwx6\" for cluster \"cluster-candidate-5-0-vdrwx6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xe7465c98060\u003e: \n failed to create NodePool np-vdrwx6: failed waiting for nodepool=\"np-vdrwx6\" for cluster \"cluster-candidate-5-0-vdrwx6\" in resourcegroup=\"rg-candidate-5-0-m92lkknqnj7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5dc4901-b409-4643-aabf-27d9e2910002\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5dc4901-b409-4643-aabf-27d9e2910002\",\n \"name\": \"b5dc4901-b409-4643-aabf-27d9e2910002\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:43:19.047485932Z\",\n \"endTime\": \"2026-10-08T02:12:26.297120954Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck to exist\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck to exist\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vdrwx6\" for cluster \"cluster-candidate-5-0-vdrwx6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xe7465c98060\u003e: \n failed to create NodePool np-vdrwx6: failed waiting for nodepool=\"np-vdrwx6\" for cluster \"cluster-candidate-5-0-vdrwx6\" in resourcegroup=\"rg-candidate-5-0-m92lkknqnj7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5dc4901-b409-4643-aabf-27d9e2910002\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5dc4901-b409-4643-aabf-27d9e2910002\",\n \"name\": \"b5dc4901-b409-4643-aabf-27d9e2910002\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:43:19.047485932Z\",\n \"endTime\": \"2026-10-08T02:12:26.297120954Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck to exist\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-6b8ck to exist\\nMachine f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-m92lkknqnj7k-candidate-vdrwx6--managed/providers/Microsoft.Compute/virtualMachines/f3w7k8p3r3o8h9q-np-vdrwx6-rs755-h28lt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e2698d9f4b5f0381bce730329f99a7610859acc4b9056cb39749233f00a8e07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"121af162046d8a171b60412751be255110229f88cfefd3ba18d5c6c164a8ae5c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25efd83de780\u003e: \n failed to create NodePool np-drrgfm: failed waiting for nodepool=\"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\" in resourcegroup=\"rg-candidate-5-1-7sfzz5wmdjlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\",\n \"name\": \"c439fc17-ab56-478e-a949-fbf669f84a7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:47:33.608428033Z\",\n \"endTime\": \"2026-10-08T02:16:36.341701384Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25efd83de780\u003e: \n failed to create NodePool np-drrgfm: failed waiting for nodepool=\"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\" in resourcegroup=\"rg-candidate-5-1-7sfzz5wmdjlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\",\n \"name\": \"c439fc17-ab56-478e-a949-fbf669f84a7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:47:33.608428033Z\",\n \"endTime\": \"2026-10-08T02:16:36.341701384Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"27bcd88ffba0420bae797a06b7e2794d3cdae8006b7a7213bf1b46b3cc2e566b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"5e6c253faadc9bb7037e12aa7cd2177efa6e4d786e05f9ec021dc0d6ca95d93e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pcfgpm\" for cluster \"cluster-candidate-4-23-pcfgpm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3576982986e0\u003e: \n failed to create NodePool np-pcfgpm: failed waiting for nodepool=\"np-pcfgpm\" for cluster \"cluster-candidate-4-23-pcfgpm\" in resourcegroup=\"rg-candidate-4-23-s9jhxk9d7hqt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/370524de-fc82-4336-87bc-65ac23926875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/370524de-fc82-4336-87bc-65ac23926875\",\n \"name\": \"370524de-fc82-4336-87bc-65ac23926875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:50:33.954027938Z\",\n \"endTime\": \"2026-10-08T02:19:36.145194017Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r to exist\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r to exist\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pcfgpm\" for cluster \"cluster-candidate-4-23-pcfgpm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3576982986e0\u003e: \n failed to create NodePool np-pcfgpm: failed waiting for nodepool=\"np-pcfgpm\" for cluster \"cluster-candidate-4-23-pcfgpm\" in resourcegroup=\"rg-candidate-4-23-s9jhxk9d7hqt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/370524de-fc82-4336-87bc-65ac23926875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/370524de-fc82-4336-87bc-65ac23926875\",\n \"name\": \"370524de-fc82-4336-87bc-65ac23926875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:50:33.954027938Z\",\n \"endTime\": \"2026-10-08T02:19:36.145194017Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r to exist\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-grk6r to exist\\nMachine z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-s9jhxk9d7hqt-candidate-pcfgpm--managed/providers/Microsoft.Compute/virtualMachines/z9r3j0z3e9p9b2j-np-pcfgpm-6qph8-qxqxf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"306d3ba966e15061386709ace4b00a63724f90b9e34f96834e4e1d714f413e48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"bcab36d182af2adfa4121f84261759d0fa33d47d8fccb1037ef350c84a20b435","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x86f41c2a240\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n \"name\": \"1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:18:56.823039116Z\",\n \"endTime\": \"2026-10-08T01:42:56.76971285Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x86f41c2a240\u003e: \n resource group \"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n \"name\": \"1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:18:56.823039116Z\",\n \"endTime\": \"2026-10-08T01:42:56.76971285Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e16b800ac434d36ffa1d42c492b02c9a13d0e3977b63166c8bc11fbd9845e06f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b0ffe0d997a0518c9dfbd1cafdc3d5fb20ffe5e91855c9c1b29b3983f2d0730","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"e5f07cf3f27270f92b263fd5d0ca25a5614664732ecf3e2c83484f30809f1bf7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pxdmtf\" for cluster \"cluster-candidate-4-20-pxdmtf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f9795690600\u003e: \n failed to create NodePool np-pxdmtf: failed waiting for nodepool=\"np-pxdmtf\" for cluster \"cluster-candidate-4-20-pxdmtf\" in resourcegroup=\"rg-candidate-4-20-pb6dpm6zm8gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ccf5247-a8a3-4025-b710-f9043bee2e39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ccf5247-a8a3-4025-b710-f9043bee2e39\",\n \"name\": \"7ccf5247-a8a3-4025-b710-f9043bee2e39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:00.030564256Z\",\n \"endTime\": \"2026-10-08T02:11:06.112048987Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn to exist\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn to exist\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pxdmtf\" for cluster \"cluster-candidate-4-20-pxdmtf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f9795690600\u003e: \n failed to create NodePool np-pxdmtf: failed waiting for nodepool=\"np-pxdmtf\" for cluster \"cluster-candidate-4-20-pxdmtf\" in resourcegroup=\"rg-candidate-4-20-pb6dpm6zm8gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ccf5247-a8a3-4025-b710-f9043bee2e39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ccf5247-a8a3-4025-b710-f9043bee2e39\",\n \"name\": \"7ccf5247-a8a3-4025-b710-f9043bee2e39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:00.030564256Z\",\n \"endTime\": \"2026-10-08T02:11:06.112048987Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn to exist\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-dk5fn to exist\\nMachine o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-pb6dpm6zm8gk-candidate-pxdmtf--managed/providers/Microsoft.Compute/virtualMachines/o4r5b1i9k3h6r0r-np-pxdmtf-6jkll-pxgpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"59a1887a6391dbad565a96906b2a04a20f7b2b2d3c45bdaf9f3c4e34c60d7496","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"14e87a585f2e303b894b77d8c959ae34784e5c6157129d38e9b5301844bb7ba0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x170b4dc75dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-gp2b86bd5hhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b07d65c-7304-4507-95e3-59630e971c0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b07d65c-7304-4507-95e3-59630e971c0f\",\n \"name\": \"1b07d65c-7304-4507-95e3-59630e971c0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:21:26.589783314Z\",\n \"endTime\": \"2026-10-08T01:50:27.09244579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qt9mw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qt9mw to exist\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qxvfr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qxvfr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qt9mw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qt9mw to exist\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qxvfr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qxvfr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x170b4dc75dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-gp2b86bd5hhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b07d65c-7304-4507-95e3-59630e971c0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b07d65c-7304-4507-95e3-59630e971c0f\",\n \"name\": \"1b07d65c-7304-4507-95e3-59630e971c0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:21:26.589783314Z\",\n \"endTime\": \"2026-10-08T01:50:27.09244579Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qt9mw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qt9mw to exist\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qxvfr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qxvfr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qt9mw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qt9mw to exist\\nMachine c4d5u2q2v8i5c3s-np-1-8654z-qxvfr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-gp2b86bd5hhp--managed/providers/Microsoft.Compute/virtualMachines/c4d5u2q2v8i5c3s-np-1-8654z-qxvfr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6d7f6324dcb715e987652d6581bac9bdf937ef396c60e514ddac6a5ac5745a64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"a84ae99184a527fede0f7b6d9bc32e2d01c9bea2f40994bc6848ccd2c629f84b","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcfd18f3b720\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r4vhjgdwsnjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n \"name\": \"d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:22:41.727850228Z\",\n \"endTime\": \"2026-10-08T02:51:47.319042789Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcfd18f3b720\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r4vhjgdwsnjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n \"name\": \"d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T02:22:41.727850228Z\",\n \"endTime\": \"2026-10-08T02:51:47.319042789Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r4vhjgdwsnjg\" to finish creating: GET https://management"}},{"row_id":"765d51ce58908c763ae8717218654bc68a3a455d20296a7f61a6f171bd15c897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"88e397f360afd6197fec72b913472ab67b4b2d10029ecc02621fb17a674f3316","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tdmg5v\" for cluster \"cluster-candidate-4-21-tdmg5v\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d1b2ad8bcc0\u003e: \n failed to create NodePool np-tdmg5v: failed waiting for nodepool=\"np-tdmg5v\" for cluster \"cluster-candidate-4-21-tdmg5v\" in resourcegroup=\"rg-candidate-4-21-4tbqvrc7sgc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\",\n \"name\": \"1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:02.12275582Z\",\n \"endTime\": \"2026-10-08T02:11:06.102672717Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc to exist\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc to exist\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tdmg5v\" for cluster \"cluster-candidate-4-21-tdmg5v\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d1b2ad8bcc0\u003e: \n failed to create NodePool np-tdmg5v: failed waiting for nodepool=\"np-tdmg5v\" for cluster \"cluster-candidate-4-21-tdmg5v\" in resourcegroup=\"rg-candidate-4-21-4tbqvrc7sgc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\",\n \"name\": \"1ac5a2fc-c57b-40ed-870a-b5372a8c7e46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:02.12275582Z\",\n \"endTime\": \"2026-10-08T02:11:06.102672717Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc to exist\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-jqcqc to exist\\nMachine g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4tbqvrc7sgc8-candidate-tdmg5v--managed/providers/Microsoft.Compute/virtualMachines/g6c8b7j7z0r2i3j-np-tdmg5v-vtsdm-spkmt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"955eeee7316161ddaf52c8346c4f904072db9c5db86c8f3582ada7ff14973326","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"0f183d4bcd814663af627cb09ff2b4f81f9b8d7b96a1ed9c5dbfef43215375ea","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c5e53f64c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p8759llzs4pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n \"name\": \"ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:18:57.884545997Z\",\n \"endTime\": \"2026-10-08T01:47:57.229221309Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n5sb8-kxqvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n5sb8-kxqvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c5e53f64c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p8759llzs4pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n \"name\": \"ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:18:57.884545997Z\",\n \"endTime\": \"2026-10-08T01:47:57.229221309Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n5sb8-kxqvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n5sb8-kxqvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b87a9b382b93e61eba4bf38817ca51fcf5a5e3716f09112b625fe4f0a7c2aede","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"2f5ae43bb07be2044a3396af722d16a0065e608923ead5bdafb775bb39e37f4c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5c6cr2\" for cluster \"cluster-candidate-4-22-5c6cr2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb83ba3deaa0\u003e: \n failed to create NodePool np-5c6cr2: failed waiting for nodepool=\"np-5c6cr2\" for cluster \"cluster-candidate-4-22-5c6cr2\" in resourcegroup=\"rg-candidate-4-22-9rjgc2ft2c5b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\",\n \"name\": \"62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:14.829363977Z\",\n \"endTime\": \"2026-10-08T02:11:16.24601066Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm to exist\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm to exist\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5c6cr2\" for cluster \"cluster-candidate-4-22-5c6cr2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb83ba3deaa0\u003e: \n failed to create NodePool np-5c6cr2: failed waiting for nodepool=\"np-5c6cr2\" for cluster \"cluster-candidate-4-22-5c6cr2\" in resourcegroup=\"rg-candidate-4-22-9rjgc2ft2c5b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\",\n \"name\": \"62c14f8c-751c-40cf-8cb7-a8c4b8573ef3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:42:14.829363977Z\",\n \"endTime\": \"2026-10-08T02:11:16.24601066Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm to exist\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-744gm to exist\\nMachine w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9rjgc2ft2c5b-candidate-5c6cr2--managed/providers/Microsoft.Compute/virtualMachines/w0z5n7c1v2q8l7j-np-5c6cr2-5jkc5-8mnld to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3f88715dc74e9f350ff3b5fb5446b5c240ef5c49445d16181c3a20e1a7f41f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","failed_at":"e2e","signature_id":"ae1a8ec85d5f87964e54e270ab9ebf79bf92e62da6f496c93f46a3a9dfd56020","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x273477d04048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zvv7lmwhhzgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n \"name\": \"81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:30:01.277978624Z\",\n \"endTime\": \"2026-10-08T01:59:06.873419943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x273477d04048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zvv7lmwhhzgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n \"name\": \"81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T01:30:01.277978624Z\",\n \"endTime\": \"2026-10-08T01:59:06.873419943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","e16b800ac434d36ffa1d42c492b02c9a13d0e3977b63166c8bc11fbd9845e06f","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}