{"meta":{"start_date":"2026-10-11","end_date":"2026-10-11","start_at":"2026-10-11T00:00:00Z","end_at":"2026-10-12T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T04:06:40Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":10,"failed_runs":10,"raw_occurrences":13,"matched_occurrences":12,"runs_affected":9},"rows":[{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":40,"anchor_occurrences":4,"after_last_push_count":73,"trend_counts":[3,6,9,3,2,1,4],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":162,"prior_last_seen_at":"2026-10-03T16:57:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4}],"full_error_samples":["alert fired 17 time(s) (14 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lhgsl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lhgsl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-tt7dl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-tt7dl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:03:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-qdz9k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-qdz9k has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-vp2rh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-vp2rh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"kube-controller-manager-6fbbbcbff9-8w5vs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/kube-controller-manager-6fbbbcbff9-8w5vs has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-dfpqp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-dfpqp has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k\", pod=\"ovnkube-control-plane-c5d7b9d44-sm4sw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k/ovnkube-control-plane-c5d7b9d44-sm4sw has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-5jm6z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-5jm6z has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hrjrj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hrjrj has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:03:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-8qzfv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-8qzfv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hp7l6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hp7l6 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-jzrt5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-jzrt5 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lvr9j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lvr9j has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-12T00:47:36Z\n Ended: 2026-10-12T01:08:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le\", pod=\"kube-controller-manager-bf5f57fb8-bdk2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le/kube-controller-manager-bf5f57fb8-bdk2x has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T00:36:32Z\n Ended: 2026-10-12T00:41:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8959360-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f\", pod=\"cluster-network-operator-67f4c84888-hzql4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f/cluster-network-operator-67f4c84888-hzql4 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T21:23:26Z\n Ended: 2026-10-11T21:29:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6435200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"f68370508ff6caa5c18b35640c5e7236383094944a4be830b451b52b18add7bd","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"a3c7d3032023cb3482ce8c92f0dd2a976d734609da54cda79a8760b5940a0595","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"24628989d0288208601920b5e851ec201bb5169aacd7861a259ad071fc8838f4","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109140137769177088","occurred_at":"2026-10-11T04:32:49Z","signature_id":"6a61843b5445b12ff0d8e5e34d69f5aab37878cf077a6ff5a1fe09ff25a81ff5","pr_number":6452,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":71,"trend_counts":[26,15,29,19,4,0,2],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":197,"prior_last_seen_at":"2026-10-04T23:41:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T18:33:37Z\n Ended: 2026-10-11T18:42:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8107776-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-11T15:52:40Z\n Ended: 2026-10-11T16:12:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1984128-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109331063628107776","occurred_at":"2026-10-11T17:11:29Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109291724151984128","occurred_at":"2026-10-11T14:35:05Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a69ffbb8a895af0747c6f4a476af227a32150aff47581bb3fd7f521ea56ce13e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109310669768626176","occurred_at":"2026-10-11T15:50:27Z","signature_id":"929e7bd719fc59b94777a6ea0a168be5415092782f3e6702397acd6adaa7c232","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,2,1,2,1,1,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"0c85b612becba4c571d22661fd122ecdb27d938385ea00a4fd07f500c5e96552","pr_number":7439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[1,1,0,1,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-02T23:57:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"b2888b7c2e1a7a0a79a19f9ced813090e269f2019ace640a3be1b766564988b0","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[5,6,3,6,2,1,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":26,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-11T23:00:46Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3354496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.245:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3354496/providers/microsoft.monitor/accounts/services-j3354496\", namespace=\"klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jhnzl\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7 on cluster ci01-j3354496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109391747573354496","occurred_at":"2026-10-11T21:12:37Z","signature_id":"128d05881de79fd499086d5693a2febe7c68869a9b1927ffc1becfa9da92767e","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[1,0,1,0,2,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-02T18:39:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"0942b9f98992dfeb8e8e398ab3ba99f81052ee6ffa88dc880de4c1cdafa43b86","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":110,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109252833235701760","occurred_at":"2026-10-11T12:00:15Z","signature_id":"f33e39eed2f7fbe6ff53acf892c123b972cdb4f2a8fa3a7d7e9f3a49cf743347","pr_number":7219,"after_last_push_of_merged_pr":false}]}]},{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":7,"matched_occurrences":7,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":99,"also_in":["STG"],"trend_counts":[2,2,2,5,5,5,4],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":24,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fc144a8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-48s6s8v68r22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: 308b2783-d950-49dc-bba1-b7f5095895de\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x2482f98dd8f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kgfhn26xlnc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: 6cddad86-b2cb-402b-b9c5-2d33d805daed\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2478de214db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9589trzrcm22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 1985dc52-0e75-413c-9ede-87cb3e6ad675\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"05c29ad480b5bf27166e88948710fee03cc002f61af8efde1454d88889eb8859","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"57f4b8a29534a1b97f7efd40c16cb7d3faa7d37950a5b8bae5e2ef89286cdd95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"85e4ca2686ea9b5410efbbf71ef6da967a97fc5a0f3281b695c388c6119013b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"cf7cb1ca2d45d1b8b3bddf2bf8a6201374e2153edb4dc1531f8dc0a98730aadf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,2],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-30T23:00:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x11944cc8930\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x11944978520\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x11944cc8930\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x11944978520\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"5bfccd42c01d56d5347db2bbce6e7db03d40121dacef58a1a5dc37e3bff7a16f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"eb1c33da48c2dd7e1a17df691460f82388ce7033187824a4224223b7cbcc8fe1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x5325624e738\u003e: \n resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x5325624e738\u003e: \n resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","occurred_at":"2026-10-11T22:05:45Z","signature_id":"7581ac8d1a8bb671ccd2233ea3ec2804c211a820c9531c5cee88d22d89400cc4","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":1,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":5,"after_last_push_count":72,"also_in":["PROD"],"trend_counts":[5,5,5,5,5,5,5],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":15,"prior_last_seen_at":"2026-10-03T23:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b2d2ab80c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 6a1cddf9-69d2-480e-9369-0bdf2d5fafea\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2b2d2ab80c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-99dhh5lb5zrf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 6a1cddf9-69d2-480e-9369-0bdf2d5fafea\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x3a9f56439c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 9de7ea40-32be-4320-949c-45a19d8c9f2a\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x3a9f56439c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nhdf6nnvkxfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 9de7ea40-32be-4320-949c-45a19d8c9f2a\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3cb7f249e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 326b9417-df61-4372-bd77-c007393853e2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3cb7f249e30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-bwnjnsrh77xp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 326b9417-df61-4372-bd77-c007393853e2\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"1ddd75a4df7db175072426fb3e32a0f46c38e701f3c10c744a66995a09746157","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"2e8e7d85409aa17236b72b19ba8a8fe1c5ab5162a21ecac980d27db3bce6fd7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"65ac56874a943c9487d98905ee69c4459ed863f90e311c5e9ddd55258d7f6cf6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"6e040c991069a028c97b26eda22de505e4ce221c866b4b12509ad5f2670e7dc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"cce4a16ca669e9b7057b8665b57073714f79b4a75f6b8a8dd1bd6abaf244a6cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-nnnfjhqw4q7g/cluster-nightly-4-20-4mtrqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e45881b1020\u003e: \n failed to create HCP cluster cluster-nightly-4-20-4mtrqs: failed waiting for cluster=\"cluster-nightly-4-20-4mtrqs\" in resourcegroup=\"rg-nightly-4-20-nnnfjhqw4q7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"name\": \"20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:20:27.297702896Z\",\n \"endTime\": \"2026-10-11T23:49:31.823735812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n correlation ID: 75d06690-940d-4212-a332-53f65ea86d06\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-nnnfjhqw4q7g/cluster-nightly-4-20-4mtrqs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e45881b1020\u003e: \n failed to create HCP cluster cluster-nightly-4-20-4mtrqs: failed waiting for cluster=\"cluster-nightly-4-20-4mtrqs\" in resourcegroup=\"rg-nightly-4-20-nnnfjhqw4q7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"name\": \"20d3ef3a-5603-49e7-a89b-176457ccafcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:20:27.297702896Z\",\n \"endTime\": \"2026-10-11T23:49:31.823735812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20d3ef3a-5603-49e7-a89b-176457ccafcd\n correlation ID: 75d06690-940d-4212-a332-53f65ea86d06\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-c9kmmf\" at install version 4.21.0-0.nightly-multi-2026-10-11-170003\nUnexpected error:\n \u003c*fmt.wrapError | 0x323f6ff48740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-c9kmmf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-c9kmmf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lds8gpglp25k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"name\": \"ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:26:08.511529037Z\",\n \"endTime\": \"2026-10-11T23:55:11.677255215Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n correlation ID: b7d51903-b039-4593-bd39-bbd75181cbd0\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-c9kmmf\" at install version 4.21.0-0.nightly-multi-2026-10-11-170003\nUnexpected error:\n \u003c*fmt.wrapError | 0x323f6ff48740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-c9kmmf: failed waiting for cluster=\"cp-ystream-upgrade-4-22-c9kmmf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-lds8gpglp25k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"name\": \"ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:26:08.511529037Z\",\n \"endTime\": \"2026-10-11T23:55:11.677255215Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec5fa575-dd1b-4347-b7d7-0d42eb1c89ee\n correlation ID: b7d51903-b039-4593-bd39-bbd75181cbd0\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"6d2bb094474926c3944a6419013f592fd6ee220fa057099f1207616936341d71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"e4f5dc1ebfa32972bb080c0f1650a448a932216fc94a43ef0b38493ce1cd05c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"994cc87ca03dcd70cb204d8d4904359ec0b786075af157e6024413d9b295804d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"cbedd273ed9a5b32964ea691b7c8952675d5b75c9ecaa831abc108bc9530aa44","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"bcf7bc5e7d129d2651e6f43b84038ce2267b5bf6cb489b6ebbbdd577ed510c1b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":26,"prior_last_seen_at":"2026-10-04T22:01:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n client request ID: unavailable"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"a72db00a2257deae0a9b15a0618c4edf13182ab5933eabd4872272d30cdc4b95","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}