{"meta":{"start_date":"2026-10-11","end_date":"2026-10-11","start_at":"2026-10-11T00:00:00Z","end_at":"2026-10-12T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T07:01:57Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":10,"failed_runs":10,"raw_occurrences":13,"matched_occurrences":12,"runs_affected":9},"rows":[{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":40,"anchor_occurrences":4,"after_last_push_count":73,"trend_counts":[3,6,9,3,2,1,4],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":162,"prior_last_seen_at":"2026-10-03T16:57:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4}],"full_error_samples":["alert fired 17 time(s) (14 unknown, 3 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:04:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lhgsl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lhgsl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-tt7dl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-tt7dl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:03:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-qdz9k\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-qdz9k has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-vp2rh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-vp2rh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"kube-controller-manager-6fbbbcbff9-8w5vs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/kube-controller-manager-6fbbbcbff9-8w5vs has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-dfpqp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-dfpqp has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:05:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k\", pod=\"ovnkube-control-plane-c5d7b9d44-sm4sw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm44q5ep2osnuu42ndqjo11p7p3m6-q6t1s1d8h2q6s5k/ovnkube-control-plane-c5d7b9d44-sm4sw has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-5jm6z\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-5jm6z has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:04:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hrjrj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hrjrj has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:03:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i\", pod=\"ignition-server-proxy-7f44d8d46-8qzfv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm576q05rv6mv1tq3kk25is17j9mp-d9q4e1z1g2p8n7i/ignition-server-proxy-7f44d8d46-8qzfv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:04:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-hp7l6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-hp7l6 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:04:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-jzrt5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-jzrt5 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas\", pod=\"ignition-server-proxy-5988489558-lvr9j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm2ma2es3e2pjmur7r5mim45j0fib-private-kas/ignition-server-proxy-5988489558-lvr9j has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-12T00:47:36Z\n  Ended:       2026-10-12T01:08:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6174848-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le\", pod=\"kube-controller-manager-bf5f57fb8-bdk2x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctm3l4c9fvor23h7j8oq4g14n444ug-ingress-cert-le/kube-controller-manager-bf5f57fb8-bdk2x has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T00:36:32Z\n  Ended:       2026-10-12T00:41:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8959360-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f\", pod=\"cluster-network-operator-67f4c84888-hzql4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tctccghfgvuis1e4ubo7a7ap2uim0kd-w0e2c0f4m6k0z7f/cluster-network-operator-67f4c84888-hzql4 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-11T21:23:26Z\n  Ended:       2026-10-11T21:29:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6435200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tcqfah109q9u6g2h4huthue530shv73-p6r1u7s0v8h7i2h/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"f68370508ff6caa5c18b35640c5e7236383094944a4be830b451b52b18add7bd","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"a3c7d3032023cb3482ce8c92f0dd2a976d734609da54cda79a8760b5940a0595","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"24628989d0288208601920b5e851ec201bb5169aacd7861a259ad071fc8838f4","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109140137769177088","occurred_at":"2026-10-11T04:32:49Z","signature_id":"6a61843b5445b12ff0d8e5e34d69f5aab37878cf077a6ff5a1fe09ff25a81ff5","pr_number":6452,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":71,"trend_counts":[26,15,29,19,4,0,2],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":197,"prior_last_seen_at":"2026-10-04T23:41:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-11T18:33:37Z\n  Ended:       2026-10-11T18:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8107776-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-11T15:52:40Z\n  Ended:       2026-10-11T16:12:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1984128-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109331063628107776","occurred_at":"2026-10-11T17:11:29Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109291724151984128","occurred_at":"2026-10-11T14:35:05Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a69ffbb8a895af0747c6f4a476af227a32150aff47581bb3fd7f521ea56ce13e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n      \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n            correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n            client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n      \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n            correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n            client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109310669768626176","occurred_at":"2026-10-11T15:50:27Z","signature_id":"929e7bd719fc59b94777a6ea0a168be5415092782f3e6702397acd6adaa7c232","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,2,1,2,1,1,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n      \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n            correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n            client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n      \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n            correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n            client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"0c85b612becba4c571d22661fd122ecdb27d938385ea00a4fd07f500c5e96552","pr_number":7439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[1,1,0,1,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-02T23:57:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n            correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n            client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n            correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n            client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"b2888b7c2e1a7a0a79a19f9ced813090e269f2019ace640a3be1b766564988b0","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[5,6,3,6,2,1,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":26,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-11T23:00:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3354496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.245:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3354496/providers/microsoft.monitor/accounts/services-j3354496\", namespace=\"klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jhnzl\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tcrhmibf7k7vglj9iommmu4f595r7e7 on cluster ci01-j3354496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109391747573354496","occurred_at":"2026-10-11T21:12:37Z","signature_id":"128d05881de79fd499086d5693a2febe7c68869a9b1927ffc1becfa9da92767e","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[1,0,1,0,2,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-02T18:39:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"0942b9f98992dfeb8e8e398ab3ba99f81052ee6ffa88dc880de4c1cdafa43b86","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-05..2026-10-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":110,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n      \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n      \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n            correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n            client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xe1c86824ac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cnc5f6qgdgz8\" to finish creating: GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\",\n      \"name\": \"58fb2269-9049-4fd2-9274-ac2190a19932\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T12:57:29.816928878Z\",\n      \"endTime\": \"2026-10-11T13:26:38.515811706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xr7ss-282dw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xr7ss-282dw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cnc5f6qgdgz8--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xr7ss-282dw to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5701760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/58fb2269-9049-4fd2-9274-ac2190a19932\n            correlation ID: b9b2478c-6a44-4ebb-ade0-4de59279ccb4\n            client request ID: 4eee6950-aeb4-430b-b956-66f798ff6c28"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109252833235701760","occurred_at":"2026-10-11T12:00:15Z","signature_id":"f33e39eed2f7fbe6ff53acf892c123b972cdb4f2a8fa3a7d7e9f3a49cf743347","pr_number":7219,"after_last_push_of_merged_pr":false}]}]}]}