{"meta":{"date":"2026-10-11","timezone":"UTC","generated_at":"2026-10-14T03:14:46Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":10,"failed_runs":10,"runs_with_occurrences":10,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T23:29:37Z","started_at":"2026-10-11T23:29:37Z","completed_at":"2026-10-12T01:50:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T02:02:08.325198583Z","timing_metadata_checked_at":"2026-10-12T02:02:08.325198583Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T02:02:08.32924217Z","region_metadata_checked_at":"2026-10-12T02:02:08.32924217Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4b6d5daa2575b82108816630a11430abc52e4ce6e7ef88109f26c524acf9a227","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","failed_at":"alert","signature_id":"f68370508ff6caa5c18b35640c5e7236383094944a4be830b451b52b18add7bd","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 17 time(s) (14 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lhgsl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lhgsl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-tt7dl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-tt7dl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:03:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-qdz9k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-qdz9k has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-vp2rh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-vp2rh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"kube-controller-manager-6fbbbcbff9-8w5vs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/kube-controller-manager-6fbbbcbff9-8w5vs has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-dfpqp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-dfpqp has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k\", pod=\"ovnkube-control-plane-c5d7b9d44-sm4sw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k/ovnkube-control-plane-c5d7b9d44-sm4sw has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-5jm6z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-5jm6z has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hrjrj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hrjrj has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:03:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-8qzfv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-8qzfv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hp7l6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hp7l6 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-jzrt5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-jzrt5 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lvr9j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lvr9j has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le\", pod=\"kube-controller-manager-bf5f57fb8-bdk2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le/kube-controller-manager-bf5f57fb8-bdk2x has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"93653093001c1cd6d12503f5ee2a6e81cec8293c90a3a7b4390d7907f0e3b05f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","failed_at":"e2e","signature_id":"0c85b612becba4c571d22661fd122ecdb27d938385ea00a4fd07f500c5e96552","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7424,"pr_state":"closed","pr_sha":"c154328295ceb9b8e8a45b91b824588993971141","final_merged_sha":"c154328295ceb9b8e8a45b91b824588993971141","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T23:05:50Z","started_at":"2026-10-11T23:05:50Z","completed_at":"2026-10-12T01:28:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T01:40:47.402518445Z","timing_metadata_checked_at":"2026-10-12T01:40:47.402518445Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T01:40:47.406043234Z","region_metadata_checked_at":"2026-10-12T01:40:47.406043234Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"bb37a67bc954e939be89e5c19f71013554fe3bb7e62c787fd2d312e2a3cbae40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","failed_at":"e2e","signature_id":"b2888b7c2e1a7a0a79a19f9ced813090e269f2019ace640a3be1b766564988b0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"e099e420f99b23863844cba8b59c90de48e35d7acd0c2ba6a2fa1ff3024e463d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","failed_at":"alert","signature_id":"a3c7d3032023cb3482ce8c92f0dd2a976d734609da54cda79a8760b5940a0595","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T00:36:32Z\n Ended: 2026-10-12T00:41:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8959360-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f\", pod=\"cluster-network-operator-67f4c84888-hzql4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f/cluster-network-operator-67f4c84888-hzql4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109391747573354496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"eebaf988e665953ea3afd1de9c05774f7fbfd235","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-11T21:12:37Z","started_at":"2026-10-11T21:12:37Z","completed_at":"2026-10-11T23:32:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T23:44:04.700405955Z","timing_metadata_checked_at":"2026-10-11T23:44:04.700405955Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T23:44:04.704018744Z","region_metadata_checked_at":"2026-10-11T23:44:04.704018744Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c94abe1aaa5b22d44bc5db089322c6106ad37456301cc7370bc5440bcb7c8115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109391747573354496","occurred_at":"2026-10-11T21:12:37Z","failed_at":"alert","signature_id":"128d05881de79fd499086d5693a2febe7c68869a9b1927ffc1becfa9da92767e","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-11T23:00:46Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3354496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.245:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3354496/providers/microsoft.monitor/accounts/services-j3354496\", namespace=\"klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jhnzl\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7 on cluster ci01-j3354496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109377888988958720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7458,"pr_state":"closed","pr_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","final_merged_sha":"5e5ec5c43047bb9fada4d13d03f843ad7621a829","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T20:17:33Z","started_at":"2026-10-11T20:17:33Z","completed_at":"2026-10-11T20:47:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T20:54:32.730800067Z","timing_metadata_checked_at":"2026-10-11T20:54:32.730800067Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-11T20:54:32.734671355Z","region_metadata_checked_at":"2026-10-11T21:57:50.693965615Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"423c719753559947750d2045383b11648fed68e929de7457ef4f950d9c39e4a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109377888988958720","occurred_at":"2026-10-11T20:17:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"closed","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T19:48:48Z","started_at":"2026-10-11T19:48:48Z","completed_at":"2026-10-11T22:32:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T22:51:30.59924721Z","timing_metadata_checked_at":"2026-10-11T22:51:30.59924721Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T22:51:30.602822799Z","region_metadata_checked_at":"2026-10-11T22:51:30.602822799Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9c94d3e60066318d83b153b33e7bd9da744be82547ed7f7068481d14831d285c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","failed_at":"alert","signature_id":"24628989d0288208601920b5e851ec201bb5169aacd7861a259ad071fc8838f4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T21:23:26Z\n Ended: 2026-10-11T21:29:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6435200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e65e28118bae45d84372cd428265de33856324e86c8f8a3358ff761c87b3a760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","failed_at":"e2e","signature_id":"0942b9f98992dfeb8e8e398ab3ba99f81052ee6ffa88dc880de4c1cdafa43b86","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109331063628107776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"closed","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T17:11:29Z","started_at":"2026-10-11T17:11:29Z","completed_at":"2026-10-11T19:32:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T19:42:41.395615609Z","timing_metadata_checked_at":"2026-10-11T19:42:41.395615609Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T19:42:41.399269691Z","region_metadata_checked_at":"2026-10-11T19:42:41.399269691Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"632b2b155434c1e2fdf72df302ed47f3ce54bb85b6a232983e718dbaebe353ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109331063628107776","occurred_at":"2026-10-11T17:11:29Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T18:33:37Z\n Ended: 2026-10-11T18:42:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8107776-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109310669768626176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"a7588ae59ddd79bdf62a22115b0a4decd0527f7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-11T15:50:27Z","started_at":"2026-10-11T15:50:27Z","completed_at":"2026-10-11T18:14:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T18:28:45.109669955Z","timing_metadata_checked_at":"2026-10-11T18:28:45.109669955Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T18:28:45.117065331Z","region_metadata_checked_at":"2026-10-11T18:28:45.117065331Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"add86f0bd3b5f4dc0b758dc2312c3f66372a2220814758a3f16d42625519e1d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109310669768626176","occurred_at":"2026-10-11T15:50:27Z","failed_at":"e2e","signature_id":"929e7bd719fc59b94777a6ea0a168be5415092782f3e6702397acd6adaa7c232","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a69ffbb8a895af0747c6f4a476af227a32150aff47581bb3fd7f521ea56ce13e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a69ffbb8a895af0747c6f4a476af227a32150aff47581bb3fd7f521ea56ce13e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109291724151984128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"closed","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T14:35:05Z","started_at":"2026-10-11T14:35:05Z","completed_at":"2026-10-11T17:09:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T17:15:46.546100292Z","timing_metadata_checked_at":"2026-10-11T17:15:46.546100292Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T17:15:46.549862179Z","region_metadata_checked_at":"2026-10-11T17:15:46.549862179Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ab03aecb784e28bbcdb26a8fdfb745ecce24550ebb739d46fc720215102d1d6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109291724151984128","occurred_at":"2026-10-11T14:35:05Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T15:52:40Z\n Ended: 2026-10-11T16:12:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1984128-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109252833235701760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7219,"pr_state":"open","pr_sha":"0e1c991ca2d9b9776ba755f464d7ccdc27554ba6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-11T12:00:15Z","started_at":"2026-10-11T12:00:15Z","completed_at":"2026-10-11T14:09:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T14:17:50.811778043Z","timing_metadata_checked_at":"2026-10-11T14:17:50.811778043Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T14:17:50.815350331Z","region_metadata_checked_at":"2026-10-11T14:17:50.815350331Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a72deba62c80b2bd4de9637cb9e95ebfe6e0d905e328b9062f1669747ef625a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109252833235701760","occurred_at":"2026-10-11T12:00:15Z","failed_at":"e2e","signature_id":"f33e39eed2f7fbe6ff53acf892c123b972cdb4f2a8fa3a7d7e9f3a49cf743347","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109140137769177088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6452,"pr_state":"open","pr_sha":"839b3d3714c21805403cf72c985814c6243aa023","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-11T04:32:49Z","started_at":"2026-10-11T04:32:49Z","completed_at":"2026-10-11T06:42:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T06:46:43.177086215Z","timing_metadata_checked_at":"2026-10-11T06:46:43.177086215Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T06:46:43.180761503Z","region_metadata_checked_at":"2026-10-11T06:46:43.180761503Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"798a2f5c0b7c6d6e631168d0e049f4d358ed3805e01c1eae8400205e3aefa280","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109140137769177088","occurred_at":"2026-10-11T04:32:49Z","failed_at":"alert","signature_id":"6a61843b5445b12ff0d8e5e34d69f5aab37878cf077a6ff5a1fe09ff25a81ff5","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 119 time(s) (91 unknown, 28 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-s44bv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-s44bv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-tspd2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-tspd2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-ztqvn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-ztqvn has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-pp7zn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-pp7zn has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-r72lv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-r72lv has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-nqrl7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-nqrl7 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-sx6j8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-sx6j8 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-vlbv7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-vlbv7 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-hjrmd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-hjrmd has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-p4d2c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-p4d2c has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-59nvf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-59nvf has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-p7kmj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-p7kmj has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"control-plane-operator-5d494df9db-kr48l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/control-plane-operator-5d494df9db-kr48l has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-xp74t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-xp74t has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-knbx9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-knbx9 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-5kklq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-5kklq has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:03:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-vqpjv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-vqpjv has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-764cz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-764cz has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-c2hxl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-c2hxl has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-k7cqj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-k7cqj has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-fkpms\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-fkpms has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-qkvcs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-qkvcs has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-5jxf6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-5jxf6 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-dz5nh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-dz5nh has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-ktmkd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-ktmkd has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-c2lp7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-c2lp7 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-g6bw4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-g6bw4 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-999w8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-999w8 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-qxvsl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-qxvsl has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-dln2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-dln2j has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-v8flq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-v8flq has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-7r6qc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-7r6qc has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-lj7wd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-lj7wd has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:03:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-vwd26\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-vwd26 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-ph8gv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-ph8gv has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-kh5zr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-kh5zr has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-6r5ln\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-6r5ln has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-xrh2q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-xrh2q has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-rmgkv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-rmgkv has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-4bd5r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-4bd5r has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-ztg86\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-ztg86 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-4twsj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-4twsj has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-bpf5r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-bpf5r has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-wgc2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-wgc2x has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-cdkhc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-cdkhc has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-vgll5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-vgll5 has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-ln4zj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-ln4zj has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-7v5qb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-7v5qb has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-4q9wz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-4q9wz has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-rxd9d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-rxd9d has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-6mwn2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-6mwn2 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-247jg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-247jg has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"control-plane-operator-5d494df9db-nl4zs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/control-plane-operator-5d494df9db-nl4zs has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-fw9c4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-fw9c4 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-lfq89\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-lfq89 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-v4jn2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-v4jn2 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-kv5s2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-kv5s2 has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-ddfxc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-ddfxc has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-blff2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-blff2 has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-jffgt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-jffgt has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-mpgpp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-mpgpp has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-wh54t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-wh54t has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-bv7qc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-bv7qc has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-2xgbl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-2xgbl has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-8bvfh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-8bvfh has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-lmvqp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-lmvqp has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-czzdk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-czzdk has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-8lkjj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-8lkjj has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-ww4c7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-ww4c7 has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-qrpjz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-qrpjz has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-568rf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-568rf has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-jt8qn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-jt8qn has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-5tr5d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-5tr5d has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-cpkmb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-cpkmb has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-7ctwg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-7ctwg has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-sb8km\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-sb8km has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-njj8b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-njj8b has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-s57sq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-s57sq has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-svt9x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-svt9x has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:01:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-fngz6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-fngz6 has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-7grwx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-7grwx has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-z56x9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-z56x9 has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-ph8p9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-ph8p9 has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-nplt8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-nplt8 has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-ht56m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-ht56m has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-766ms\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-766ms has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:02:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-bmp96\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-bmp96 has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:03:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a\", pod=\"cluster-api-758886dcdd-tdcb8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdt5p5krgk8c9s2qa52muf0s2leo-o5c6e9o7o9u8u2a/cluster-api-758886dcdd-tdcb8 has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T06:06:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q\", pod=\"ovnkube-control-plane-84746c8bb7-7k2zp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctg51ab7lhn1327cqn1ro610k7o09-g7m4x8h2o0n2u0q/ovnkube-control-plane-84746c8bb7-7k2zp has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:57:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p\", pod=\"network-node-identity-699479db79-hc79n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcctdihs6bopliddcb2r3r2grv2jd6r-f0j1t3p3q3h3e5p/network-node-identity-699479db79-hc79n has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n State: Resolved\n Started: 2026-10-11T05:38:35Z\n Ended: 2026-10-11T05:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9177088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s\", pod=\"kube-controller-manager-7d88d8ddc6-wdfkq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcct8i6p9rmb6p0qtjmra4rleu9p1p6-y7r8u9b9d9y6s8s/kube-controller-manager-7d88d8ddc6-wdfkq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T23:00:04Z","started_at":"2026-10-11T23:00:04Z","completed_at":"2026-10-12T00:45:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T00:58:30.478354889Z","timing_metadata_checked_at":"2026-10-12T00:58:30.478354889Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T00:58:30.483428574Z","region_metadata_checked_at":"2026-10-12T00:58:30.483428574Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"36774bf613da331c47957ed4a013004a9780cdf2def3be9d058d7e4aa4f96bcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"eb1c33da48c2dd7e1a17df691460f82388ce7033187824a4224223b7cbcc8fe1","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"4f0ac55f5eef4de629970d6f2c5f86c6b0b7b040cf64e0e25ab50c6936ad94d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"85e4ca2686ea9b5410efbbf71ef6da967a97fc5a0f3281b695c388c6119013b0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5ffb6d180876e3414b5bd73485bf79991420fef7da5484a0ab20aed8647ae9b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"57f4b8a29534a1b97f7efd40c16cb7d3faa7d37950a5b8bae5e2ef89286cdd95","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"62c0dfa14bc9c329ffadf183946ead2294b6040c8d2ca37d8d155714f83505ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"05c29ad480b5bf27166e88948710fee03cc002f61af8efde1454d88889eb8859","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bc7522491dec07e326d00739fe6b3d2c44c566618715cfdb9f43b4010d7c7725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"5bfccd42c01d56d5347db2bbce6e7db03d40121dacef58a1a5dc37e3bff7a16f","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x11944cc8930\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x11944978520\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x11944cc8930\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x11944978520\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"d9caa32413ed00b32c9351ebd45e879765d815f8e33f796b2cc9925739d980b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"cf7cb1ca2d45d1b8b3bddf2bf8a6201374e2153edb4dc1531f8dc0a98730aadf","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x1809749e4ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 169674f2-a45c-486a-9ed8-ef274e1803f2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x1809749e4ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-bt79q5smg9ln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 169674f2-a45c-486a-9ed8-ef274e1803f2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T22:05:45Z","started_at":"2026-10-11T22:05:45Z","completed_at":"2026-10-12T00:42:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T00:58:30.251434754Z","timing_metadata_checked_at":"2026-10-12T00:58:30.251434754Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T00:58:30.255498242Z","region_metadata_checked_at":"2026-10-12T00:58:30.255498242Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d14675bde38ebe616c7cb9c10b3d5e8956cef554a7c8f4e5682094648004f4f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","occurred_at":"2026-10-11T22:05:45Z","failed_at":"e2e","signature_id":"7581ac8d1a8bb671ccd2233ea3ec2804c211a820c9531c5cee88d22d89400cc4","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x5325624e738\u003e: \n resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x5325624e738\u003e: \n resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2109101693110063104","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-11T02:00:03Z","started_at":"2026-10-11T02:00:03Z","completed_at":"2026-10-11T04:19:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T04:30:06.66084184Z","timing_metadata_checked_at":"2026-10-11T04:30:06.66084184Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T04:30:06.697334727Z","region_metadata_checked_at":"2026-10-11T04:30:06.697334727Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-11T23:00:04Z","started_at":"2026-10-11T23:00:04Z","completed_at":"2026-10-12T00:39:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-12T00:58:30.659782958Z","timing_metadata_checked_at":"2026-10-12T00:58:30.659782958Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-12T00:58:30.662801949Z","region_metadata_checked_at":"2026-10-12T00:58:30.662801949Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0ebffba9d8353f28a482a9f5351f1674f6ba2f5de2e5bde01fb6045b8228b4b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"a72db00a2257deae0a9b15a0618c4edf13182ab5933eabd4872272d30cdc4b95","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n client request ID: unavailable","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management"}},{"row_id":"3a98ab99535ecc757eadc1a615f85ff9984b449f79fdee6ba1a503f0e4832169","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"65ac56874a943c9487d98905ee69c4459ed863f90e311c5e9ddd55258d7f6cf6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3cb7f249e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 326b9417-df61-4372-bd77-c007393853e2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3cb7f249e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 326b9417-df61-4372-bd77-c007393853e2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"54d66d67fc7e1f6fc0d29f77844e0d34209d1b253b0d21323248cd6799df566d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"cce4a16ca669e9b7057b8665b57073714f79b4a75f6b8a8dd1bd6abaf244a6cd","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x651b1589410\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gx6bw8krnxdc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gx6bw8krnxdc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 8266feca-7a35-4c9d-b5da-450588c39766\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x651b1589410\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gx6bw8krnxdc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gx6bw8krnxdc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 8266feca-7a35-4c9d-b5da-450588c39766\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"676a3a08074c7311ee9596bb4efb1503c8a032d194d31a00283c439d5e2b1b96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"994cc87ca03dcd70cb204d8d4904359ec0b786075af157e6024413d9b295804d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aac8d2dc6d37400cf8b4a195e9d7d3b8adbe7de3d031ec515d24cde2e9372085","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"1ddd75a4df7db175072426fb3e32a0f46c38e701f3c10c744a66995a09746157","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b2d2ab80c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 6a1cddf9-69d2-480e-9369-0bdf2d5fafea\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b2d2ab80c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 6a1cddf9-69d2-480e-9369-0bdf2d5fafea\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b8c24272435de11179f3e967a48322aa236992fd8571256ff53f1061f6011050","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"6e040c991069a028c97b26eda22de505e4ce221c866b4b12509ad5f2670e7dc3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c69c127620\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6bqc7wxnpls2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6bqc7wxnpls2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 8c15b69a-1a45-4b77-97f7-cc3c4f7303d9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c69c127620\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6bqc7wxnpls2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6bqc7wxnpls2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 8c15b69a-1a45-4b77-97f7-cc3c4f7303d9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c81ad2ed22085f3a949b9f238147ff146b2926fcd3edea1efadc1b2bdd1b24d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"2e8e7d85409aa17236b72b19ba8a8fe1c5ab5162a21ecac980d27db3bce6fd7f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x3a9f56439c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 9de7ea40-32be-4320-949c-45a19d8c9f2a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x3a9f56439c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 9de7ea40-32be-4320-949c-45a19d8c9f2a\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e69a18e862b8385bfea1163a6fcb5c5c6135f30a789cdbef239844cbbb57e680","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"bcf7bc5e7d129d2651e6f43b84038ce2267b5bf6cb489b6ebbbdd577ed510c1b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cbedd273ed9a5b32964ea691b7c8952675d5b75c9ecaa831abc108bc9530aa44","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f4d9820b6c39277198d4d6d645de9803339d270c14c5d20ff2c5f4b196787f20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"6d2bb094474926c3944a6419013f592fd6ee220fa057099f1207616936341d71","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-nnnfjhqw4q7g/cluster-nightly-4-20-4mtrqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e45881b1020\u003e: \n failed to create HCP cluster cluster-nightly-4-20-4mtrqs: failed waiting for cluster=\"cluster-nightly-4-20-4mtrqs\" in resourcegroup=\"rg-nightly-4-20-nnnfjhqw4q7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"name\": \"20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:20:27.297702896Z\",\n \"endTime\": \"2026-10-11T23:49:31.823735812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n correlation ID: 75d06690-940d-4212-a332-53f65ea86d06\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-nnnfjhqw4q7g/cluster-nightly-4-20-4mtrqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e45881b1020\u003e: \n failed to create HCP cluster cluster-nightly-4-20-4mtrqs: failed waiting for cluster=\"cluster-nightly-4-20-4mtrqs\" in resourcegroup=\"rg-nightly-4-20-nnnfjhqw4q7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"name\": \"20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:20:27.297702896Z\",\n \"endTime\": \"2026-10-11T23:49:31.823735812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n correlation ID: 75d06690-940d-4212-a332-53f65ea86d06\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f9aeb0db425a0e3963d8e4a2e39b785e4e9dc8d526c183d081d4a7adf8ab35f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","failed_at":"e2e","signature_id":"e4f5dc1ebfa32972bb080c0f1650a448a932216fc94a43ef0b38493ce1cd05c9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-c9kmmf\" at install version 4.21.0-0.nightly-multi-2026-10-11-170003\nUnexpected error:\n \u003c*fmt.wrapError | 0x323f6ff48740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-c9kmmf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-c9kmmf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lds8gpglp25k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"name\": \"ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:26:08.511529037Z\",\n \"endTime\": \"2026-10-11T23:55:11.677255215Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n correlation ID: b7d51903-b039-4593-bd39-bbd75181cbd0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-c9kmmf\" at install version 4.21.0-0.nightly-multi-2026-10-11-170003\nUnexpected error:\n \u003c*fmt.wrapError | 0x323f6ff48740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-c9kmmf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-c9kmmf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lds8gpglp25k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"name\": \"ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:26:08.511529037Z\",\n \"endTime\": \"2026-10-11T23:55:11.677255215Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n correlation ID: b7d51903-b039-4593-bd39-bbd75181cbd0\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","cbedd273ed9a5b32964ea691b7c8952675d5b75c9ecaa831abc108bc9530aa44"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2109101693164589056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-11T02:00:03Z","started_at":"2026-10-11T02:00:03Z","completed_at":"2026-10-11T03:55:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-11T03:58:37.539283641Z","timing_metadata_checked_at":"2026-10-11T03:58:37.539283641Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-11T03:58:37.542990929Z","region_metadata_checked_at":"2026-10-11T03:58:37.542990929Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}