{"meta":{"start_date":"2026-10-22","end_date":"2026-10-28","start_at":"2026-10-22T00:00:00Z","end_at":"2026-10-29T00:00:00Z","timezone":"UTC","generated_at":"2026-10-28T23:57:02Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":473,"failed_runs":308,"raw_occurrences":577,"matched_occurrences":540,"runs_affected":271},"rows":[{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":25,"runs_affected":25,"failed_runs":25,"run_impact_percent":5.2854122621564485,"anchor_occurrences":25,"after_last_push_count":1,"trend_counts":[2,6,3,2,4,4,2,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":27,"prior_last_seen_at":"2026-10-25T17:30:28Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":25}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T13:27:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.125:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-phdkm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5471b0ac-6a29-48cb-ab86-2893cd772fbd\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-phdkm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T13:29:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-84g47\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9462daaa-47d3-4b5e-b73b-01a9f857a137\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-84g47 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T13:29:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-84g47\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9462daaa-47d3-4b5e-b73b-01a9f857a137\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-84g47 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T13:32:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0229248-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.125:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0229248/providers/microsoft.monitor/accounts/services-j0229248\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-phdkm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5471b0ac-6a29-48cb-ab86-2893cd772fbd\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-phdkm (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T10:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.26:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-5dmcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6520e185-94fe-45c4-99a9-813a1698d907\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-5dmcb (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T10:43:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-ts4n7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a28f799-e0bd-4698-b612-8c6756c4b517\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-ts4n7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T10:43:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-ts4n7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a28f799-e0bd-4698-b612-8c6756c4b517\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-ts4n7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T10:47:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1445504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.26:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1445504/providers/microsoft.monitor/accounts/services-j1445504\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-5dmcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6520e185-94fe-45c4-99a9-813a1698d907\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-5dmcb (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T08:13:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-krfd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2298911c-0832-493f-a9ef-f65728646e11\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-krfd7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T08:13:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-krfd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2298911c-0832-493f-a9ef-f65728646e11\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-krfd7 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T08:15:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.208:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-fgmvz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e461328c-3664-4b8e-907f-f8f42456ba46\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-fgmvz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T08:15:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3004160-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.208:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3004160/providers/microsoft.monitor/accounts/services-j3004160\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7fc5986cf8-fgmvz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e461328c-3664-4b8e-907f-f8f42456ba46\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7fc5986cf8-fgmvz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","signature_id":"5c1f0d2c18530e902522efe7242b86668fe04b921a7be0312c5fd330b98f0748","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376885591445504","occurred_at":"2026-10-28T09:35:25Z","signature_id":"f7f309c518710e27e7a8f712e71c207623d203053f761a3188198d64808a5e96","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"dfd7afbd88dab1923cf763897f9f70a5cd1bd457893585b2411046095b00387c","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"6717050d02899506dc19d396fc04fde0a13d0f143390e00656ec8a22fd39225c","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","signature_id":"4c14651213a33fb215c6ba3326174190c0648e3ebef08c277144d262c0761f29","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","signature_id":"33415ae06c670660cd19c33a16e1db7f18b2fe42419fc9b5576f8033ad4489ab","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","signature_id":"cece8a7f4a2f9a33b54d6b23adca1a5ac146b55ffe47ba10a70c28b7a7652d75","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114650067662016512","occurred_at":"2026-10-26T09:27:19Z","signature_id":"615406ebab48e14110a4e9d8f4e52090829600500bb8dd3e92ffc884098b6345","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114613195741073408","occurred_at":"2026-10-26T07:00:36Z","signature_id":"875a87e1cceb054032906ab6545693da41bff75ce6526371ca7a798456a52fa2","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","signature_id":"1d391c5e1f8d80343056e901525b045ccb36fa0af9d1f6801f1f896f98f95d70","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"23a900339c164e91dcce00d9aeb9286d19def65ff9ea3201cee586844076595c","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333714598072320","occurred_at":"2026-10-25T12:30:14Z","signature_id":"10b7f95e300ee35e6c9a51a0b80079fc89d7716cc4d6f052cfff2bc326ebbb9f","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"0a897df4c8f15692a53a241d7a951370aa0b2ed5d1e9e6678c0fa066f18fee55","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"83f9f91a80bdac318fe991b6d8496f6250de9707233dd99d4cad0f40c175e591","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850692501245952","occurred_at":"2026-10-24T04:30:53Z","signature_id":"4092fa71e6585c9f0469827fb61244698d5b97435db774acf42bc2d72d8bb3a3","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795606852931584","occurred_at":"2026-10-24T00:51:59Z","signature_id":"7fe27bdbd1b619b6b49df4650d8476278c6f6132c533ba3ef57f974fe7697567","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","signature_id":"b6d85dbc1d4502551493043fa33721464a3b1282465557d679ae70789ff0b520","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2ec5e8a05ec938e5d3691a4dec2f741db485c232fb0b69893230211b7263938c","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"fd738f0b860459440540292342939e3ce110921d918faf32d78e2b8f0cbc73ec","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","signature_id":"6ec942e8ee5fad5d5c4fba3fbbe4fe73778eae33cb5e869a5d69169f1367dfe6","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","signature_id":"b5976c7e247238d154539db68b0098f770efffac46ccd361fcbee9ff68b1cf23","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113208030127460352","occurred_at":"2026-10-22T09:57:10Z","signature_id":"8217ab2784f36f979beaba44454a1e2325f5d27c7a000ec1e76e7eb24e5b4ea1","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","signature_id":"a8a3c6a30c55341f0d2bcacbcbbd3a03912b87ddffb2ee938913151c553a557f","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","signature_id":"5022976043c0a5c67816fadc890268fa3ab94e3bd6c81af70a2cfa059aa64b1c","pr_number":7675,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","signature_id":"3f2c87dd944435486cb7cb53c2d27ce5177997363dcef8154e66da2bd6dddf52","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":23,"runs_affected":23,"failed_runs":23,"run_impact_percent":4.862579281183932,"anchor_occurrences":23,"after_last_push_count":13,"trend_counts":[0,0,0,0,0,1,22,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":23,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":23}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.004s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.008s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.005s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019604114804736","occurred_at":"2026-10-27T09:55:43Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":6635,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019570476486656","occurred_at":"2026-10-27T09:55:35Z","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","pr_number":6823,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008102154637312","occurred_at":"2026-10-27T09:10:01Z","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","pr_number":7514,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008051160289280","occurred_at":"2026-10-27T09:09:48Z","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","pr_number":7770,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114994824758169600","occurred_at":"2026-10-27T08:17:15Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":7740,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6714/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114993082914050048","occurred_at":"2026-10-27T08:10:20Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":6714,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114987666448584704","occurred_at":"2026-10-27T07:48:48Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982798447087616","occurred_at":"2026-10-27T07:29:28Z","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982484201443328","occurred_at":"2026-10-27T07:28:11Z","signature_id":"bac0e3e75226f5af3bd5b014a630521a1cfaa2655fa7d3ded74b5c1381f24c77","pr_number":7041,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7761/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977910874968064","occurred_at":"2026-10-27T07:10:02Z","signature_id":"6a821c50d92310b73e1cf552308326b196dd980765b78ba09ceed32b40ce48d3","pr_number":7761,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114977151651418112","occurred_at":"2026-10-27T07:07:01Z","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","pr_number":7770,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975739370541056","occurred_at":"2026-10-27T07:01:08Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":7769,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975256258023424","occurred_at":"2026-10-27T06:59:30Z","signature_id":"17d4cdfc77354f622fc0109e20b904a9def3d63396c30a68d9e99bbbfc413481","pr_number":7768,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","signature_id":"13be5e1341477c52eade788cb598f1cc4e1ab24ef4f65e65ff9ddcdba8af23bb","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114964013585534976","occurred_at":"2026-10-27T06:14:49Z","signature_id":"6451a4f5c64d9abbc32284dc078828b1f9e659530fba80fcbdd390788556e7fa","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114952950173405184","occurred_at":"2026-10-27T05:30:51Z","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","pr_number":7740,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","signature_id":"dce5e5ce5976358e3c7a925a1fbd158f7e68fc614b9e70512494514a648c8b2a","pr_number":7767,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","signature_id":"728610916b6bb6aabd8b6387fd2165a99c6b1300da63f977405000424541f334","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918518389477376","occurred_at":"2026-10-27T03:14:02Z","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114886591380983808","occurred_at":"2026-10-27T01:07:10Z","signature_id":"eb4791c627f0b323cd84778d6ab630dcf68cf79863d86304a1ae217fade01c89","pr_number":7739,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114870474402959360","occurred_at":"2026-10-27T00:03:08Z","signature_id":"a55b324811ab43c0a5e26983bf074b2fb5d7459add9bd7edaf2846d3703aa47f","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7766/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114861755761954816","occurred_at":"2026-10-26T23:28:29Z","signature_id":"13be5e1341477c52eade788cb598f1cc4e1ab24ef4f65e65ff9ddcdba8af23bb","pr_number":7766,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":3.8054968287526427,"anchor_occurrences":18,"after_last_push_count":30,"trend_counts":[11,4,3,0,1,4,5,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":146,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":18}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T20:00:50Z\n  Ended:       2026-10-28T20:06:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5001984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le\", pod=\"openshift-apiserver-7cb4d98fc-rzx96\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le/openshift-apiserver-7cb4d98fc-rzx96 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T23:40:09Z\n  Ended:       2026-10-27T23:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1933056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 13 time(s) (12 unknown, 1 known)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:44:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le\", pod=\"openshift-apiserver-5fbf969589-rwzww\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le/openshift-apiserver-5fbf969589-rwzww has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:44:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b\", pod=\"openshift-apiserver-7c75f5b8c-869mc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b/openshift-apiserver-7c75f5b8c-869mc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w\", pod=\"openshift-apiserver-6cf589c77c-9cn7v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w/openshift-apiserver-6cf589c77c-9cn7v has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t\", pod=\"openshift-apiserver-9c9cfdf89-62q4b\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t/openshift-apiserver-9c9cfdf89-62q4b has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u\", pod=\"openshift-apiserver-7bbccfbcd8-nj4zh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u/openshift-apiserver-7bbccfbcd8-nj4zh has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x\", pod=\"openshift-apiserver-7b79cf8d6d-klztg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x/openshift-apiserver-7b79cf8d6d-klztg has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:45:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u\", pod=\"openshift-apiserver-7d75b894c6-n4cmx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u/openshift-apiserver-7d75b894c6-n4cmx has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le\", pod=\"openshift-apiserver-764b69ffcb-7ngbg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le/openshift-apiserver-764b69ffcb-7ngbg has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a\", pod=\"cluster-policy-controller-698bfd9c65-v6xdj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a/cluster-policy-controller-698bfd9c65-v6xdj has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f\", pod=\"openshift-apiserver-57bf4dc46f-h7mj5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f/openshift-apiserver-57bf4dc46f-h7mj5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:46:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k\", pod=\"openshift-apiserver-59b86f6c67-76jwk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k/openshift-apiserver-59b86f6c67-76jwk has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:47:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1476864-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g\", pod=\"openshift-apiserver-77c77b89b8-kr9pl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g/openshift-apiserver-77c77b89b8-kr9pl has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509651545001984","occurred_at":"2026-10-28T18:22:59Z","signature_id":"980846bb94448eb2630686e4469bd6f85bb6d9608d00cf33c8152c9e5c035322","pr_number":7814,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","signature_id":"5f52e62e8a632dc6425c669a8d6a48a88e99ab3ee83dc1b80737d8ba7f2924f7","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115087800381476864","occurred_at":"2026-10-27T14:26:42Z","signature_id":"1dedc5282ccef457cd2bfffb95d080a0e8a4f79e607e2268fb8bfe663ae779f0","pr_number":7730,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115031737661657088","occurred_at":"2026-10-27T10:43:56Z","signature_id":"232640154453a0f874248e0023fa85692463590a9f33b666fe19b05a6e973c88","pr_number":7441,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115028667607289856","occurred_at":"2026-10-27T10:31:44Z","signature_id":"836edef393ea769d8c1dbbd5c1835e62c2571df370eab8875c778843c628400d","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115019495591383040","occurred_at":"2026-10-27T09:55:17Z","signature_id":"e051fbf77b070edcec4d4461b179266cc25bf3709a7c61aa952c8a230f477902","pr_number":7730,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7766/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114861755761954816","occurred_at":"2026-10-26T23:28:29Z","signature_id":"8a6cfe1f3b994f21ecf195a4cfd956a5b5ecbe8f556e74114048743a0dc4001d","pr_number":7766,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","signature_id":"d89e640561546ab0437c91630c89df761f8f4b3267b8d41e85d378d9d14fc187","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","signature_id":"090e7f849d686e279e1fdb0a19cd25c6885887022d3ccec9aff0064e35b748f0","pr_number":7758,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","signature_id":"4c00445100490919aaaa60858ad67d7b978d873b864c379367d94dc07300f433","pr_number":7721,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"c88b83725f4c290056abe9fc249d58727c9801758819fd002dadddfc47f86675","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113628897605062656","occurred_at":"2026-10-23T13:49:33Z","signature_id":"762381bb4f426bcce14d2dcedf2af2dc5db6afa5d64699c2dabcc9907a483ca0","pr_number":7717,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"6fbf7b7865ca254c8d602bde28984e80b575972637dde5ddb43c09fae006779f","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"b85e109c2f4dc5d74b9ee622dcf47c821f416845163d08b78b0f76a24ed1a9fa","pr_number":7687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","signature_id":"dec7456054680cbb2852e9fb295189edbf59f26b803821d887eace607491b311","pr_number":7548,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","signature_id":"1acc704dec9da24c5c7fa9f35ac9616ae74f220571f4a3427e7537e172296446","pr_number":7653,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113256572250492928","occurred_at":"2026-10-22T13:10:03Z","signature_id":"278d0b8c7c257830ef45bc0ace5ac701b95b0ade44e6e541f1c00a6415a0baaf","pr_number":7633,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113171725192531968","occurred_at":"2026-10-22T07:32:54Z","signature_id":"e8bab8e8bd10f29c86040fc7faea61d020bbc0e4e4e89f6d21a0bf4619b7714f","pr_number":7373,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":18,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":18,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[1,0,12,6,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-24T12:30:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x11da4f3f5de0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x11da4f3f5de0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x13fb9bc14400\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x13fb9bc14400\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x274951b76fe0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x274951b76fe0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","signature_id":"a44daeee5e71bfd01ffb050800873ed56d216ca4254ff105a3f77a6cfeb167b0","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","signature_id":"a8d7c33431d2d5e1c043c56c17b798c04ea45500f5120f8e441080c92ba01b8a","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"95e59c2767f43e6f5ccf12ae6495ea4c7859921f5efc5fc067b0fbd45a7fcefd","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"b45772d9c21eeb78596ea8932a4abcae55a029b0f156a15bd528b50bbacb64a2","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","signature_id":"5a97ec00b2365f6302a91eeb09a77807c49fff76d3d016fc94f23dcb1ee35688","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","signature_id":"b84151051ec5b2adf25427fb8178337ecae54b7aa256dc8f766dcc6c6a454410","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"06077bac2bdfbefc430aa21e7f1c63585e4b0facfbed8535434aa7846c098a94","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"150ce84edc5920033c79f1d71e7933469c3d65ffd64a306a05fe2dabbc654d50","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"5d631566a7a8e50a23a29e16a0e18527e365de9f11bacc81791b4a3fa0dd48df","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"6a69757f3cbee730cf9d60087714cca23dc702792cdaa339a7014d9b67fd1143","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"70670de7f9371c033d82071ca8d86a5ee534cd8acfcecfb1faac595d80a44997","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"712adee26418d9b97d2614e8f26ca63599b26688a67ad28c5cce4a2daa4c4e84","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"905453ac6e335acb4f35cd240fa3b7536659d0d95ac9186a5592efab0f975eea","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"99fcfb8a0f2194c211d91e98d4874fd0b7609259139ff0a43afeb62267c42148","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"b79149b9fc941d47e42641fbc00f500cd063ac6efa35986bbc24fa7b5d878613","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"bb412fd2d485854280b1c9d07b101b3e663f2d9f084fff735671240db5800f4f","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"d879674894167409266496657d54be06c8aa7f724945913ba06a1ba076ac2290","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"f75c1d43c7a4c5c09304ce65eeb484f3c9012f635c820f63f75d2054e572fc61","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":17,"runs_affected":14,"failed_runs":14,"run_impact_percent":2.959830866807611,"anchor_occurrences":17,"after_last_push_count":8,"also_in":["STG"],"trend_counts":[0,2,6,1,0,4,3,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":142,"prior_last_seen_at":"2026-10-24T00:55:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2db4fc89320\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zbnm2gmmrc7k\" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n      \"name\": \"13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T16:52:57.688694522Z\",\n      \"endTime\": \"2026-10-28T17:12:06.752450543Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n            correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc\n            client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2db4fc89320\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zbnm2gmmrc7k\" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n      \"name\": \"13b873cd-2efa-4559-ab15-ff37ab2d031b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T16:52:57.688694522Z\",\n      \"endTime\": \"2026-10-28T17:12:06.752450543Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b\n            correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc\n            client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb71d4fca080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bqbx2znvzb54\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"name\": \"0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:25:38.076776565Z\",\n      \"endTime\": \"2026-10-27T23:44:40.197052822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc\n            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb71d4fca080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bqbx2znvzb54\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"name\": \"0ed006bd-b8aa-4770-9062-3fd090d4fea9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:25:38.076776565Z\",\n      \"endTime\": \"2026-10-27T23:44:40.197052822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9\n            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc\n            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3591356d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"name\": \"e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:27:39.482004964Z\",\n      \"endTime\": \"2026-10-27T23:46:40.23395218Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d\n            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3591356d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6b928vvvc9lv\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"name\": \"e68fe219-929c-499f-9d94-52a89aa54331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:27:39.482004964Z\",\n      \"endTime\": \"2026-10-27T23:46:40.23395218Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331\n            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d\n            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115467457228443648","occurred_at":"2026-10-28T15:35:19Z","signature_id":"6ec2a0a084a83fcca9f0f2578989c9f523c37f120d4f7b1d92bf237362af37ee","pr_number":7695,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","signature_id":"0a0ea7133cf05c130127f02fb8aeee975c38e805f3657e70069a7cb13aced4f0","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","signature_id":"e39e3016c6f3cedf1cbe0ad7596d7aa46a070ea53b8406bc85c701f66772306e","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122591927635968","occurred_at":"2026-10-27T16:44:57Z","signature_id":"565fcf6646d614a2f928329c9d0fc3e5adde16bae5d4e208002a59dbb76d5fe4","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114600535943286784","occurred_at":"2026-10-26T06:10:29Z","signature_id":"37d6b1898665834c194cc9382c6cb7f22a57b414b03e07c41ba2411d31830fda","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114600535943286784","occurred_at":"2026-10-26T06:10:29Z","signature_id":"e3ae56b421b472adb90d5bf2cff478a4dadb50c8f4068e47ee19cfa9e72139da","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","signature_id":"2cc1fbf964c09132d4e1288a799526694dfa03c7ea3a5312bc375046fdc651c1","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","signature_id":"5b1a358ec60ee2fc7fa42a8d358fb8f4ff780a44d4fe2ab5e89289cd2283efed","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796458485387264","occurred_at":"2026-10-24T00:55:22Z","signature_id":"2c4af7aec4f24afaa73fdc991afb8658d4ba85d9f78d068284914a918c56028a","pr_number":7730,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113768338508222464","occurred_at":"2026-10-23T23:03:38Z","signature_id":"186c2f62dbba5354fdd50e3f198e2f5c7260e536efa09a9dba4dd236da4ad630","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113714121043087360","occurred_at":"2026-10-23T19:28:05Z","signature_id":"bb2f64cac1081e7e6bb8f566f3bd4023215ffc8e942c129ba96fe59be1320761","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"edc4a2a86804f16bb2ca5ef5fec66418b27ca460d596142f26648f4d317f99da","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","signature_id":"d9f385324d27a77fd6938089006a621965c3093828acb8f05519d964d111d9ed","pr_number":7073,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113542145401950208","occurred_at":"2026-10-23T08:02:16Z","signature_id":"d04d5f67613fcded3e78320b3bd7ce7a8186accddad999333f00e221da713c03","pr_number":7658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"5efae9732adc28512b06dafb9969fee7b2e6984fa695d99435471c1675c6ced2","pr_number":7463,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113319770949423104","occurred_at":"2026-10-22T17:21:11Z","signature_id":"96f835c54af083d55baeaf346ab65584abf7bcf116d908e62aac2a124faa4aac","pr_number":7696,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113076587715891200","occurred_at":"2026-10-22T01:14:52Z","signature_id":"83d43091dcf6c3fabb39c58f779510f09c9b8b29f38f6f33dd93bee3852e5924","pr_number":6772,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":16,"runs_affected":16,"failed_runs":16,"run_impact_percent":3.382663847780127,"anchor_occurrences":16,"after_last_push_count":22,"also_in":["INT","PROD","STG"],"trend_counts":[0,3,2,0,1,2,3,5],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":61,"prior_last_seen_at":"2026-10-25T12:30:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a2a28c8a260\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h8dst6v865z6\" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n      \"name\": \"455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:16:02.73250898Z\",\n      \"endTime\": \"2026-10-28T20:45:04.251585998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n            correlation ID: 582a61df-2172-42e6-962b-b544533d3da6\n            client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a2a28c8a260\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-h8dst6v865z6\" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n      \"name\": \"455ab4ea-ab03-4a58-91c6-5ad5406405ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:16:02.73250898Z\",\n      \"endTime\": \"2026-10-28T20:45:04.251585998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff\n            correlation ID: 582a61df-2172-42e6-962b-b544533d3da6\n            client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x370d5101eb20\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tgjxrh8mcrw6\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\",\n      \"name\": \"cce1648d-4b31-48bb-9884-f170bb5391eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:28:55.409225524Z\",\n      \"endTime\": \"2026-10-28T19:58:03.222551854Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n            correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e\n            client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x370d5101eb20\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tgjxrh8mcrw6\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\",\n      \"name\": \"cce1648d-4b31-48bb-9884-f170bb5391eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:28:55.409225524Z\",\n      \"endTime\": \"2026-10-28T19:58:03.222551854Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb\n            correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e\n            client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ee0c1834d20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster=\"np-version-upgrade-cluster-jl2sj8\" in resourcegroup=\"rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf\" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\",\n      \"name\": \"267d3daf-f468-46eb-af3a-f1b72d573583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:42:27.649991944Z\",\n      \"endTime\": \"2026-10-28T20:11:28.996573337Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n            correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23\n            client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ee0c1834d20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster=\"np-version-upgrade-cluster-jl2sj8\" in resourcegroup=\"rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf\" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\",\n      \"name\": \"267d3daf-f468-46eb-af3a-f1b72d573583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:42:27.649991944Z\",\n      \"endTime\": \"2026-10-28T20:11:28.996573337Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583\n            correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23\n            client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","signature_id":"c5700ceb14bbf3145f2fe9d4d95f47c4875ea9e5983e09584bafc3d75fbf7935","pr_number":7666,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"60a0140959ffc2ca32cabe626c237c34f8d6c30d556a56d5f6cfe1fa2ef7e592","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115506901079822336","occurred_at":"2026-10-28T18:12:04Z","signature_id":"4b1c26a0d10e93a5af6942386f37009deed677c3e114f68b9bdcf149db621ce3","pr_number":7704,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","signature_id":"fc014753f356fcc9708c367a6543a1d77e1f8177081d9e11992a0453189c6596","pr_number":7782,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"ee364ce84dd90a160b7dc0e8bc2d5885b02b4ac45c05e55def20833e767b8add","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","signature_id":"4a9d64725d8b6bbd0f0f8918c4d21324df99b2aeb2db108ba6a317b2bf3e34ad","pr_number":7705,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","signature_id":"d36af26ba43900f37b1a8934a7c91a186aab3388d1da85c98b9b4b581315a6c9","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"847c23767aa51d34dbcbd44bfae866c902845ee3aca8c91c84f1387745ed29c9","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","signature_id":"9e335c0a832d5ff50fcfdbb223c4a042675de44458234ab8586e50623b976b80","pr_number":7758,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","signature_id":"b5e430d12aac3714e721d5dd8023ef20e2860c7fe2979215316742856535fb70","pr_number":7739,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"acc8285070bd989dffae9a81dcf632a7bf3a9db3d381a859414064975ec30cc3","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2871aba458073a19748dc486b5fa832486d5dacb815d2e076d0ba71820b99e15","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"7e722ad5a4ed9ab1a5b378b6eaaa935525e7b9867c93ed401690a68dd2ba6792","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"37fb4527b0dca28de2500cb1a873ec11aeee90e63a01f0b3975e6a14eb0ed6fc","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","signature_id":"38380caeb49ac20d9aa623cb9f89aeb6a3a057f42abe6a005618f92bc4b435b1","pr_number":7683,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","signature_id":"11bb1183f5742505aad8eb8162c6762f75c32bdf1650bcffd71b6e3d63f7d81c","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":16,"runs_affected":16,"failed_runs":16,"run_impact_percent":3.382663847780127,"anchor_occurrences":16,"after_last_push_count":22,"trend_counts":[0,2,3,0,2,3,2,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":63,"prior_last_seen_at":"2026-10-25T12:30:03Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":16}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T20:39:20Z\n  Ended:       2026-10-28T21:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"777a1913-78dc-4e11-9844-4f50d6675327\"\n  Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T20:40:18Z\n  Ended:       2026-10-28T21:06:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"777a1913-78dc-4e11-9844-4f50d6675327\"\n  Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T19:50:44Z\n  Ended:       2026-10-28T20:18:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7\"\n  Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T19:50:44Z\n  Ended:       2026-10-28T20:19:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7\"\n  Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T12:59:31Z\n  Ended:       2026-10-28T13:28:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5496ebfd-2ab4-43b8-9f9a-e0483581f4bd\"\n  Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T13:00:33Z\n  Ended:       2026-10-28T13:27:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"5496ebfd-2ab4-43b8-9f9a-e0483581f4bd\"\n  Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","signature_id":"22cb4c1c74c288852b46936e29daa10fb7e56baf99940791080b2bc27593091c","pr_number":7666,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"da5e4122e4c53523b9d3713667cdf04b91bd9de900409d5886b3e4d9e6217e9e","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","signature_id":"1c2cf09809fe509ac5ca6af7dd0c8a42eed9a467c897a7f2bb1d533ef52c7350","pr_number":7782,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"451b2e969d1680468440ceb90c398cbbb1804c59a23c31a9dd08adaa504f000a","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","signature_id":"6651f88688a55750821d5c7f03935d0d70244cf6e6fd344d7bf85123f2fcc04f","pr_number":7705,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"090543014f12d2faca82ee316f5e8a3161876854e37b7b7810c6f5097d0e8fb6","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","signature_id":"1cdfef7346400f71d7654ce8017a3db8d59770ee09e2fd2f34ac9179bfc979ed","pr_number":7758,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","signature_id":"f7150ebe4442c37aa1818a3aec9c86f34135b9ac48a16e972bc9d66865dcbf28","pr_number":7739,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","signature_id":"b5098cc6e3c36d2fdc8d9c2477b1cb87d5baf078e5312b4abdd7b8111d538f8c","pr_number":7721,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"ec01e424b2be613766af87aa7ffd1f1f917c01aba43f8485f49377c09097c201","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"3fc3c66c8b9815981c4e1256d8181a4ecc52a448d74e0588729208f7e0320945","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"5f39e806b7aca5a0dc9a2e9d8787423dbc4725456fa3be8c785dbeffe0c743c5","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"ed02ebe0267da497eb0e3be41998c6fcb596d452363ba5bd7181d595e7764413","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"19ad7954ab7f29b25a54f19b9d0544d2410dbddc79a38cc95515845245055d71","pr_number":7463,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","signature_id":"d11f1ee63b9c35d1ba118fc95457b088065166fe3c0142c76f58ceb21d2523bc","pr_number":7683,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","signature_id":"2387f1827a61b63e936473078ea28c130b72c4d492dfe00685aff4e57613ecb7","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":3.171247357293869,"anchor_occurrences":15,"after_last_push_count":37,"trend_counts":[1,3,3,0,1,2,2,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":109,"prior_last_seen_at":"2026-10-25T12:30:03Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":15}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T20:52:20Z\n  Ended:       2026-10-28T21:03:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T20:52:20Z\n  Ended:       2026-10-28T21:03:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T20:03:50Z\n  Ended:       2026-10-28T20:15:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T20:03:50Z\n  Ended:       2026-10-28T20:15:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T13:12:32Z\n  Ended:       2026-10-28T13:28:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","signature_id":"721f8230921d7beffc6045136cc6ccc09c8b2023501ae51dd3d280a96a067666","pr_number":7666,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"87b5b22f77bfe3999c0381b1dc3ddb0ccc1c296599755520fd01de4f586c3961","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","signature_id":"54df2b15da5ef9a1db45a3687ecb60f32af1e59282713521b839e992bda2ba7a","pr_number":7782,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"d336eb4e6e6fc7c7bf769701f966b2fef1dc4f8dc73afdeb5d6c75eddd6e7a7a","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","signature_id":"d52355b973893083e742037f672c037fc0a285c782d00e94d81974354b675606","pr_number":7705,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"482db436f84040b93006731be0326af709da86d17e4d0ef8870af1a8f0485a6b","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","signature_id":"ea43d16b0dcc97d1db198e586b0982e6b5bd3e77e0dbce7ea5b1108644f712fd","pr_number":7758,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","signature_id":"09e52265a08fddd83906616cecc930c8b6e12730b4e3ecb6f278281f9c17c195","pr_number":7739,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"c478c5b907afdf2bf8291383bc1b8fdf82880ef962724d36b9c0e9c4f0ffc1c1","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2286281296e2588a0e23ff286161d911dea899212cf719d1a69d41fc8aea0670","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"b9051e9b8ccf6adbe45aedab22f7090fbb17021fbbdd41caae75f1e8b7eda5f2","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"56a88ed15c81d66a2d6b20e1497719151538162e99edd29f52fd5232a43bb042","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","signature_id":"0b4904eba35f2ce4fb4e61a5ba5bc50afccce305efed6874c4167b62c103c66b","pr_number":7683,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","signature_id":"dc0b14c2e44620ae1c288ecad48b6c8dec7c73d0f88905afa45ae94448860d82","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","signature_id":"394a5ae97331321eda54df56d575a203a99c7f35f5b568580d9d5fb95b350d96","pr_number":7675,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":14,"runs_affected":14,"failed_runs":14,"run_impact_percent":2.959830866807611,"anchor_occurrences":14,"after_last_push_count":12,"trend_counts":[0,4,1,2,0,0,6,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":57,"prior_last_seen_at":"2026-10-24T16:12:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":14}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T17:07:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-64b56768b9-r7v9d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-64b56768b9-r7v9d has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T17:07:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"fleet\", pod=\"fleet-controller-7b9596cccb-g9thf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod fleet/fleet-controller-7b9596cccb-g9thf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T17:07:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-admin-api\", pod=\"admin-api-6d56468ffd-t9snj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-t9snj has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T17:07:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3195648-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-admin-api\", pod=\"admin-api-6d56468ffd-rm24l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-rm24l has been in a non-ready state for longer than 5 minutes.","alert fired 21 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-ck87t\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-ck87t has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-n68qf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-n68qf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-bx6p7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-bx6p7 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-6fggw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-6fggw has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000002-249kw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000002-249kw has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-f2bs8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-f2bs8 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-pwgk9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-pwgk9 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-sb9gv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-sb9gv has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000001-z2292\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000001-z2292 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-sgh48\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-sgh48 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"aks-istio-system\", pod=\"azure-service-mesh-istio-cni-addon-node-jkt84\", region=\"westus3\", severity=\"warning\"\n  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-jkt84 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-42508117-vmss000000-b6bzq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-42508117-vmss000000-b6bzq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:28:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-p7rcj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/hubble-relay-58594b5dd7-p7rcj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-8696d46cbc-gcq7v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-8696d46cbc-gcq7v has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-8696d46cbc-7tl64\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-8696d46cbc-7tl64 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-6bc6657c6b-nc5dx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-6bc6657c6b-nc5dx has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-6wzh8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-6wzh8 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-rsmsf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-rsmsf has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"ocm-cs-db-7594d5ccfc-vd2b7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/ocm-cs-db-7594d5ccfc-vd2b7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-567b46758d-hzqfx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod clusters-service/clusters-service-567b46758d-hzqfx has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:30:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8270720-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-6bc6657c6b-stp2q\", region=\"westus3\", severity=\"warning\"\n  Description: Pod maestro/maestro-6bc6657c6b-stp2q has been in a non-ready state for longer than 5 minutes.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-controller-859484fc49-qhntv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/ocm-controller-859484fc49-qhntv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"cluster-manager-c5d6c587f-fd8rf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-fd8rf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"multicluster-engine-operator-79c768d748-lg8vd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-lg8vd has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-applier\", pod=\"kube-applier-8475cd7c99-dfzv8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-applier/kube-applier-8475cd7c99-dfzv8 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"ocm-webhook-68676899fd-mklv5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/ocm-webhook-68676899fd-mklv5 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6dc8498b6f-ntd7x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod mgmt-agent/mgmt-agent-6dc8498b6f-ntd7x has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"multicluster-engine\", pod=\"managedcluster-import-controller-v2-6cb46794f7-bzb25\", region=\"westus3\", severity=\"warning\"\n  Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-bzb25 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9877504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"secret-sync-controller\", pod=\"secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478705303195648","occurred_at":"2026-10-28T16:20:01Z","signature_id":"376cf6b8a607ae70f7c8bc27bdfad5a3d4307ff50aa806c754ca4f6361ee4e31","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","signature_id":"df8c4cf1c0145523d4e17617fc5c14945c699a13507fb64077340994373e075b","pr_number":5244,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085218409877504","occurred_at":"2026-10-27T14:16:27Z","signature_id":"38d8ce206a673cd02519d29467557fc74a9f952f547b41255aa497ffa5a704f3","pr_number":6823,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","signature_id":"d6ade9de695f38631520c2d467f5de31adc88dacbcbba03901a349e2aae5baa8","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069745341403136","occurred_at":"2026-10-27T13:14:57Z","signature_id":"78df0de3512493987330c17664e0954e9157f8c822af2c10ada0cac833eb8371","pr_number":6954,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115063427364818944","occurred_at":"2026-10-27T12:49:51Z","signature_id":"7ac72e7ab408d127706a4fc13bb3def720f1a28e42e40b0f980c057f98624718","pr_number":6823,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114964013585534976","occurred_at":"2026-10-27T06:14:49Z","signature_id":"ad318a0839ef635cdd7da0dbb7b359b7c65595221514e368750bb1954fa66624","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114027174305468416","occurred_at":"2026-10-24T16:12:09Z","signature_id":"0192b4d7195c586e436e00e86981f587ebb031c4ce49ca09161b7c9b266d7d44","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"913267547937bb237929f26f32e2f1afb807ffb9a402d1608c34f6a1cfe7cb6c","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","signature_id":"28d90db273f3dc28418a9638298084d32843c4ff21548aa85701ba2afb5f93dc","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113385464588144640","occurred_at":"2026-10-22T21:42:14Z","signature_id":"0a5c75aa0c2fe74940638b97aa581fdd9a1b679cd6ad35f756ff074c101cad2c","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305954912047104","occurred_at":"2026-10-22T16:26:17Z","signature_id":"a400f34dd5d9c6546d3c51f97584489f4f8247fb2e77291e277aa95514d5afd7","pr_number":7688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254849540788224","occurred_at":"2026-10-22T13:03:13Z","signature_id":"51f0bcb0b5ece926328642189c37e30f72dcfc0519d66c414435e5df060a778f","pr_number":7682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113228155324993536","occurred_at":"2026-10-22T11:17:08Z","signature_id":"447150ec04c57cb895574cbf66f0db98c26d67b4a9ca7846df88f7bff32541fa","pr_number":7641,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","occurrences":14,"runs_affected":14,"failed_runs":14,"run_impact_percent":2.959830866807611,"anchor_occurrences":14,"after_last_push_count":2,"trend_counts":[0,3,6,0,4,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-25T09:28:34Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","occurrences":8}],"full_error_samples":["time=2026-10-27T15:06:00.351Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-27T15:06:00.927Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-27T15:06:00.927Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:06:01.327Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-27T15:06:01.636Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-27T15:06:01.636Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:06:01.765Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-27T15:11:07.858Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-27T15:11:07.859Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-25T10:27:22.014Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-25T10:27:22.644Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-25T10:27:22.644Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:27:22.918Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-25T10:27:23.279Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-25T10:27:23.279Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:27:23.445Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-25T10:32:28.984Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-25T10:32:28.985Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-25T08:14:44.299Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-25T08:14:45.051Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-25T08:14:45.051Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:14:45.565Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-25T08:14:45.984Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-25T08:14:45.984Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:14:46.156Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-25T08:19:54.254Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-25T08:19:54.254Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085218409877504","occurred_at":"2026-10-27T14:16:27Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":6823,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114287994985779200","occurred_at":"2026-10-25T09:28:34Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114257867568582656","occurred_at":"2026-10-25T07:28:51Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114242927562592256","occurred_at":"2026-10-25T06:29:29Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114223810998702080","occurred_at":"2026-10-25T05:13:31Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113771192396550144","occurred_at":"2026-10-23T23:14:58Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113646623203528704","occurred_at":"2026-10-23T14:59:59Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113597591521333248","occurred_at":"2026-10-23T11:45:03Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113544934832214016","occurred_at":"2026-10-23T08:15:49Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113432151310471168","occurred_at":"2026-10-23T00:47:45Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113406116942581760","occurred_at":"2026-10-22T23:04:18Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305954912047104","occurred_at":"2026-10-22T16:26:17Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113228155324993536","occurred_at":"2026-10-22T11:17:08Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7641,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":2.748414376321353,"anchor_occurrences":13,"after_last_push_count":17,"trend_counts":[0,2,2,0,1,2,2,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":54,"prior_last_seen_at":"2026-10-25T12:30:03Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":13}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T20:37:24Z\n  Ended:       2026-10-28T21:04:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T20:37:24Z\n  Ended:       2026-10-28T21:01:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1157504-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.172:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T19:48:54Z\n  Ended:       2026-10-28T20:14:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T19:48:54Z\n  Ended:       2026-10-28T20:14:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8717568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.254:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T12:57:34Z\n  Ended:       2026-10-28T13:23:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T12:57:34Z\n  Ended:       2026-10-28T13:23:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115522964811157504","occurred_at":"2026-10-28T19:15:51Z","signature_id":"648e77fd094ec15da0a086a524b7c1d4b0cc0eb05bbe4ba4df4fe5e22225b252","pr_number":7666,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"2b3dcdce11438bf15723387608c36e111821479ddc9eeeea7696800a37119f7d","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","signature_id":"91827343ea1b42f538edbc36cb3bf02b91bb944d583eaf93fff5d26da344f32e","pr_number":7782,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"16049e988aa116aa463a1a9115799a6e2d39f159b5b564e13f1f9a9c31226285","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175056668626944","occurred_at":"2026-10-27T20:13:15Z","signature_id":"9ab9f28587da746e56d2ab1ae50303307906627a29629f6be6120283acdd04ee","pr_number":7705,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"7daeb4e2c89121c0d6a0276aae92631419d4fb62396974536d0feb3f60deb191","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","signature_id":"6060a41a5ff89f5748174d141ab758dd9bfd296792310bd5f6e435f59c60e104","pr_number":7758,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","signature_id":"aab1d38cc546b720a4e4328212fd7ca1d1fffb8160414a15cb499298469c7377","pr_number":7739,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"3198e55fd62b864e78aeda30d29dbd665bc483504cacc93fd6ae861431463810","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"bf583365df567c4528941bfba0cb382d9ef8ebcb3950c01907cf643a249725b2","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"4b46ae0c9f8b5ffa392131441a033fd3d6fba4c8a02a7cf64403d6e16934d074","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113198633527545856","occurred_at":"2026-10-22T09:19:50Z","signature_id":"ba4aa3115f35ce0988c053065b4de6eb875806daaccb54e29cea6b49686f0130","pr_number":7683,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113147384883056640","occurred_at":"2026-10-22T05:56:11Z","signature_id":"f45f5c96ff8e6ceb8e57b7dda4c8c05437e4ee5fb4582519685c9718a6396ed6","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":2.3255813953488373,"anchor_occurrences":11,"after_last_push_count":4,"also_in":["INT","PROD"],"trend_counts":[0,2,1,1,2,0,4,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":36,"prior_last_seen_at":"2026-10-25T17:30:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-678df7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d18a6e97e00\u003e: \n    DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2d18a6c74540\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-678df7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d18a6e97e00\u003e: \n    DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2d18a6c74540\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqnk2.apps.aro.c7e9m0p1o3l4d4j.d5kj.j8717568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-rvmpld\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aa0726d2620\u003e: \n    DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aa071c17780\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:241]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-rvmpld\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aa0726d2620\u003e: \n    DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aa071c17780\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ffznf.apps.aro.j5p0x2q6g6t1g2c.gabn.j8373120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cc8b3a2b3a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1cc8b3559460\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cc8b3a2b3a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1cc8b3559460\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-czc2x.apps.aro.v20260901.yl4k.j4071168.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"1a2246634d007df6ee517467969a7df4a5c707e703c29c130bd6ae72b247e256","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174872618373120","occurred_at":"2026-10-27T20:12:42Z","signature_id":"2bba532e021c86d2ed01a12b037a58a40fc2823f364697d5f4bcfc466e421a66","pr_number":7769,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115165748904071168","occurred_at":"2026-10-27T19:36:27Z","signature_id":"5b6c70c60af9afcab97207b8cdf90bb3550c811d6b46f808c94720b90fbcfe2e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","signature_id":"526f904cbbc8ad7301bdb1bf3d4e1debe9adef61ef836cb69e29184539b3a4f6","pr_number":7770,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114994824758169600","occurred_at":"2026-10-27T08:17:15Z","signature_id":"00fdd034291919a7a10b2949bec9498d190584b612339286dbbc735668b40e48","pr_number":7740,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"903e4210ba2623c82b6cf4e8df84550127761ed552ffd7f0056a59a23fd671cb","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"105d71756967099a5d9e4fd8392e89856fa8c1773b28ba775bdc10aba128134f","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114047539169726464","occurred_at":"2026-10-24T17:33:05Z","signature_id":"0f91636213bd7ade3c925588546db5d56e98c1789eec3db993e11328cd2e8d45","pr_number":7730,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"6541df12b028842ffaa548f5c83c0ed141122c488181c09a33e556c71ac2d3e8","pr_number":7463,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113419724510466048","occurred_at":"2026-10-22T23:58:22Z","signature_id":"0fc78f7bb3db5f1e5303acea86ceb6e26fde838e0d4e3a4175410040cf2516f9","pr_number":7665,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113216309620641792","occurred_at":"2026-10-22T10:30:03Z","signature_id":"4efbb868e90e0c981d31b779e618a262978897a74c8152d02034b9b5aced9b10","pr_number":7598,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":2.3255813953488373,"anchor_occurrences":11,"after_last_push_count":18,"also_in":["PROD"],"trend_counts":[3,5,2,2,0,1,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":40,"prior_last_seen_at":"2026-10-24T04:30:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d4aea46a060\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-85bttqbvwzdg\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"name\": \"7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:47:01.433391882Z\",\n      \"endTime\": \"2026-10-27T18:06:02.631484435Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n            correlation ID: b503cd11-01c7-4b49-bd8f-b11bdfa64e3b\n            client request ID: 8e981bdf-718f-4374-bc9d-9fe858083611\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d4aea46a060\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-85bttqbvwzdg\" to finish creating: GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"name\": \"7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:47:01.433391882Z\",\n      \"endTime\": \"2026-10-27T18:06:02.631484435Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-85bttqbvwzdg--managed/providers/Microsoft.Compute/virtualMachines/b3p4u5e0h1k8y7f-nodepool-128gib-6n7h6-rw99j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7635968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c4e2323-4cc6-4875-b033-1cc27f1dc9c9\n            correlation ID: b503cd11-01c7-4b49-bd8f-b11bdfa64e3b\n            client request ID: 8e981bdf-718f-4374-bc9d-9fe858083611\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x75e04e9f9a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-lmxnc7\" in resourcegroup=\"rg-np-version-upgrade-lmxnc7-q9mf48fq597k\" to finish creating: GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"name\": \"d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:25:47.925761101Z\",\n      \"endTime\": \"2026-10-26T16:44:52.193515957Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n            correlation ID: f7b1fc07-103a-435d-9acb-75f17e0334fb\n            client request ID: 9e0fce66-a04c-49ec-96a9-4c1b9377e626\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x75e04e9f9a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-lmxnc7\" in resourcegroup=\"rg-np-version-upgrade-lmxnc7-q9mf48fq597k\" to finish creating: GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"name\": \"d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:25:47.925761101Z\",\n      \"endTime\": \"2026-10-26T16:44:52.193515957Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n            correlation ID: f7b1fc07-103a-435d-9acb-75f17e0334fb\n            client request ID: 9e0fce66-a04c-49ec-96a9-4c1b9377e626\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f6cce913900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b559ft\" in resourcegroup=\"rg-np-version-upgrade-b559ft-j4qdwfs6szxx\" to finish creating: GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\",\n      \"name\": \"489dec27-e4f4-49eb-a330-6052b204ad8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T05:53:26.952860579Z\",\n      \"endTime\": \"2026-10-24T06:12:28.455904126Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n            correlation ID: d5af14b9-8e48-4974-af98-abb8c215a6d4\n            client request ID: 1283d9b2-c265-4f0a-97ea-52c82d8cd670\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f6cce913900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b559ft\" in resourcegroup=\"rg-np-version-upgrade-b559ft-j4qdwfs6szxx\" to finish creating: GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\",\n      \"name\": \"489dec27-e4f4-49eb-a330-6052b204ad8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T05:53:26.952860579Z\",\n      \"endTime\": \"2026-10-24T06:12:28.455904126Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n            correlation ID: d5af14b9-8e48-4974-af98-abb8c215a6d4\n            client request ID: 1283d9b2-c265-4f0a-97ea-52c82d8cd670\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115122591927635968","occurred_at":"2026-10-27T16:44:57Z","signature_id":"cdf4ff3359680941ca4519be04a9ab48255bff7cfae013bdf33380df365386e8","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114736549408018432","occurred_at":"2026-10-26T15:10:57Z","signature_id":"6cd1d0f68d1e9c651ce9d74f373226e952ca947646d805fdb880bdd4112f9954","pr_number":7688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","signature_id":"93d4f78b327358bdeb0108f6eae0945f79cb06676ae17e59a0390dca843aefea","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850601719730176","occurred_at":"2026-10-24T04:30:31Z","signature_id":"73e414d801e26a3b9d8e077db7186e53205ae2c6f4de6a0577eb3a964c24a61f","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"ff19091ea6b6fd108dd7780cdd577078ceaa836eeffb4b21c27d5ef11633d976","pr_number":7687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"72e291739086d9b63e2a0faa5afedbb7022d6a44908bff0d2b6ef6a7ca8a9ab5","pr_number":7709,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113315750507712512","occurred_at":"2026-10-22T17:05:13Z","signature_id":"4df1aac1e0f3c92964f69b87100586a1ffec75be0df556e93f996e421e8d3ddf","pr_number":7119,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7680/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251396953313280","occurred_at":"2026-10-22T12:49:29Z","signature_id":"0d53882633db1d5048190df3d0bfbb3d29e675e5960c5c44e74bd8e56f0ba890","pr_number":7680,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113210171659390976","occurred_at":"2026-10-22T10:05:39Z","signature_id":"f2dcdc168e6795af87723ba65fde77f18a504cb3af7a59e39b6eea863e2044ba","pr_number":6823,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","signature_id":"2a31cf35da94c36d9c9a054be4d1fc4a8f126cdb012d93530f303b9096768ee7","pr_number":7665,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113138915056553984","occurred_at":"2026-10-22T05:22:32Z","signature_id":"ff7400b28734fc76d943e06d455e076ff28cea9a80dbcc5f0df78307a659662d","pr_number":7679,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":2.3255813953488373,"anchor_occurrences":11,"after_last_push_count":61,"trend_counts":[8,2,1,1,0,0,5,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":241,"prior_last_seen_at":"2026-10-24T00:41:58Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":11}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T21:09:13Z\n  Ended:       2026-10-28T21:26:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8906368-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T19:42:52Z\n  Ended:       2026-10-28T20:02:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3991936-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T20:20:10Z\n  Ended:       2026-10-27T20:29:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7981312-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533269758906368","occurred_at":"2026-10-28T19:56:50Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7757,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509692493991936","occurred_at":"2026-10-28T18:23:09Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7785,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7436/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115159601987981312","occurred_at":"2026-10-27T18:59:37Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7436,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7776/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115086995314184192","occurred_at":"2026-10-27T14:23:30Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":7776,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040287725195264","occurred_at":"2026-10-27T11:17:54Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7769,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7767,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114888160881152000","occurred_at":"2026-10-27T01:13:24Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113793086860562432","occurred_at":"2026-10-24T00:41:58Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":6772,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113536694941650944","occurred_at":"2026-10-23T07:43:10Z","signature_id":"aeb0e7cf0f7fb6668a74127ede7587f34beff19ae9034d243e6f7865af3d860f","pr_number":7073,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113086770697474048","occurred_at":"2026-10-22T01:55:20Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":2.1141649048625792,"anchor_occurrences":10,"after_last_push_count":2,"also_in":["INT"],"trend_counts":[0,0,2,3,0,1,2,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-24T18:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-nxx9xr\"\nUnexpected error:\n    \u003c*errors.joinError | 0x138a2d1f8048\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x138a2cdf5020\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x138a2cdf5000\u003e{\n                    msg: \"failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x138a2cdf4fe0\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x138a2cd0dcc0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-nxx9xr\"\nUnexpected error:\n    \u003c*errors.joinError | 0x138a2d1f8048\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\": nodes \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x138a2cdf5020\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x138a2cdf5000\u003e{\n                    msg: \"failed to label node e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x138a2cdf4fe0\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\": nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x138a2cd0dcc0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"e3a5t5r1l8c7f8q-np-nxx9xr-qn466-6zltb\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-x4d58d\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2a036e0a6030\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\": nodes \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2a036cc0d0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x2a036cc0d080\u003e{\n                    msg: \"failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x2a036cc0d060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x2a036cc3c780\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-x4d58d\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2a036e0a6030\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\": nodes \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2a036cc0d0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x2a036cc0d080\u003e{\n                    msg: \"failed to label node t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x2a036cc0d060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\": nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x2a036cc3c780\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"t5n3k9o3y9v8w3y-np-x4d58d-tdn4x-htxvc\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-j22hl8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x237cb4aa4138\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\": nodes \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x237cb4952040\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x237cb4952020\u003e{\n                    msg: \"failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x237cb4952000\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x237cb49b60a0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-j22hl8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x237cb4aa4138\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\": nodes \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x237cb4952040\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x237cb4952020\u003e{\n                    msg: \"failed to label node s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x237cb4952000\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\": nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x237cb49b60a0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"s9t1y1s2c1s6e5d-np-j22hl8-cg7vv-84h99\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7816/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512185487626240","occurred_at":"2026-10-28T18:33:03Z","signature_id":"cce94bd2f5b67e5767a91df9d741230c8f8b592a2ad3598176e71b1aeada192a","pr_number":7816,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","signature_id":"1ba167ed521beb385a26eb7e8a0f9de2df4f2d56b11f3ae2b5ab78e067c2563c","pr_number":7786,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115092150784888832","occurred_at":"2026-10-27T14:43:59Z","signature_id":"052f4a60cc44a4e395bee8de5a9587dc7636eecf78c2988f477162a1f081c064","pr_number":6718,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115036855358459904","occurred_at":"2026-10-27T11:04:06Z","signature_id":"55f92aabdfd5dc2b9a9243a121a5e6b1cc2cbbedea71dec47917cd4058b78750","pr_number":7771,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114776506176638976","occurred_at":"2026-10-26T17:49:44Z","signature_id":"f783a774c26e0e408f8fd53ae4f7628289675d69eccb5955b7816986c2ccbbe6","pr_number":7615,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114058315167174656","occurred_at":"2026-10-24T18:15:54Z","signature_id":"e495e0c4e0c96db7a7c9023b14417bd105afffe14e2c1a410c49514991caf469","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"bcc53a189fc6e12ef22dd0c7ca1d120bdf9fb075fbbfca7a88dac32ca4fd6864","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797113543397376","occurred_at":"2026-10-24T00:57:58Z","signature_id":"d8f417e401cde35fdf1ac72973ebbf582471bdc57ba9ddd78dbb706c8363d414","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683564749918208","occurred_at":"2026-10-23T17:26:46Z","signature_id":"aa04e18b7fbd233f6ca4b1e47d82cf0953e640b1a2ab6e85b1475ab4208f342f","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113603186051780608","occurred_at":"2026-10-23T12:07:11Z","signature_id":"4cf06d6c9bb7b85aaa08e454a9e64270b9208afcbca137d23650a0ab18180e76","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":2.1141649048625792,"anchor_occurrences":10,"after_last_push_count":4,"trend_counts":[1,6,1,1,1,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-25T17:30:28Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":10}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:32:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:33:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-25T19:03:55Z\n  Ended:       2026-10-25T19:10:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-25T19:03:55Z\n  Ended:       2026-10-25T19:09:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-25T19:03:55Z\n  Ended:       2026-10-25T19:10:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3671936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.162:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3671936/providers/microsoft.monitor/accounts/services-j3671936\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-7vdl7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tlvp310lq0lqg3o9bn0eeqgoijvib5c/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-24T10:00:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-24T10:00:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-24T10:09:36Z\n  Ended:       2026-10-24T10:18:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-24T10:09:36Z\n  Ended:       2026-10-24T10:18:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-24T10:10:38Z\n  Ended:       2026-10-24T10:21:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-24T10:10:38Z\n  Ended:       2026-10-24T10:21:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","signature_id":"c57a7b8056a3891d85d45284445adb09d9edf7c2d1c098329ce32936845aefb2","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"8803add155979278d8c0e23a00c75b642794b1490db265041a71fc2a6c5a337f","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"a845c02ffa47495d53a03c9cf21fb8d837d26d6fb56147af4cab6babc202e39f","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"91e41adf8a4cbf20601080ac6e87fc658cc44668486f116ba1401163a81cca7b","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","signature_id":"609a4e6d360d5c5dfe5d49e536501c3a4adecf726fdbd6fbad2ca3bce77a4013","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254479959691264","occurred_at":"2026-10-22T13:01:44Z","signature_id":"8b832a372ecfb2ace8d8d9be81e7f2b7575ca31564d8dc52d41ae47c9b508996","pr_number":7133,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113208030127460352","occurred_at":"2026-10-22T09:57:10Z","signature_id":"d9f7e014ea9393bf34d8e64426450dfc14c29fac5bb560d8f15e8301b474387c","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","signature_id":"7a54f01ba59cb3554a88a9d5a612ad65fae371543aa6288021249b5c68542874","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113136134002315264","occurred_at":"2026-10-22T05:11:29Z","signature_id":"7f9d9381a76886da9647cfc4773d130bc904bc278913f3ad683332e01b2b5ea2","pr_number":7675,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","signature_id":"402cc04f24dfc6e5de29603c238a7b273c241bcac32793b0bf818ddb0ad63603","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":9,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4799154334038056,"anchor_occurrences":9,"after_last_push_count":16,"also_in":["PROD"],"trend_counts":[2,0,5,0,1,2,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":35,"prior_last_seen_at":"2026-10-25T09:55:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f0aa4bea20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-wj8x9hp7bqx5\" to finish creating: GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"name\": \"6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:38.679552409Z\",\n      \"endTime\": \"2026-10-27T08:28:40.674204457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n            correlation ID: 51afa422-8d59-4334-91a3-bc90ade1f885\n            client request ID: ebb6ee5a-68e9-481e-8fab-49a3ebe79568\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19f0aa4bea20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-wj8x9hp7bqx5\" to finish creating: GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"name\": \"6a73d0d2-8fd5-4af1-adbb-f385ed243d20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:38.679552409Z\",\n      \"endTime\": \"2026-10-27T08:28:40.674204457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8023424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a73d0d2-8fd5-4af1-adbb-f385ed243d20\n            correlation ID: 51afa422-8d59-4334-91a3-bc90ade1f885\n            client request ID: ebb6ee5a-68e9-481e-8fab-49a3ebe79568\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2d970cc94090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rx9rnfvmj655\" to finish creating: GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"name\": \"c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T09:38:13.907922394Z\",\n      \"endTime\": \"2026-10-26T09:57:20.458727782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n            correlation ID: 4c993536-8323-4301-a24e-f6918a189628\n            client request ID: e13d57f0-4312-4271-8e8e-02240879d813\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2d970cc94090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rx9rnfvmj655\" to finish creating: GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"name\": \"c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T09:38:13.907922394Z\",\n      \"endTime\": \"2026-10-26T09:57:20.458727782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n            correlation ID: 4c993536-8323-4301-a24e-f6918a189628\n            client request ID: e13d57f0-4312-4271-8e8e-02240879d813\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f2532896c0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n22957ncbgjt\" to finish creating: GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"name\": \"ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:58:11.830961438Z\",\n      \"endTime\": \"2026-10-26T04:17:16.025648143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n            correlation ID: 5b299ccf-a19f-4517-91e6-0d46d8d87839\n            client request ID: 050413c1-cc10-4b65-a905-bd6edbbaa97f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f2532896c0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n22957ncbgjt\" to finish creating: GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"name\": \"ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:58:11.830961438Z\",\n      \"endTime\": \"2026-10-26T04:17:16.025648143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n            correlation ID: 5b299ccf-a19f-4517-91e6-0d46d8d87839\n            client request ID: 050413c1-cc10-4b65-a905-bd6edbbaa97f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114975256258023424","occurred_at":"2026-10-27T06:59:30Z","signature_id":"fd1c1e45c21971c8bd7dc5c6ce80f975b1981c85f1832b8a925316f35d943923","pr_number":7768,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114635485509849088","occurred_at":"2026-10-26T08:29:22Z","signature_id":"2df52a6d277b5dfd6e04438bec8a6c80649d92df6621c452eb3aa83a82cb7d5f","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114552936569769984","occurred_at":"2026-10-26T03:00:55Z","signature_id":"02fba510a7bd5f759b6d77e3aaafa60e37757c068ea5f489e06ec884b3999289","pr_number":7615,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294657641877504","occurred_at":"2026-10-25T09:55:02Z","signature_id":"a8e2229b6df36d2c0aba95f4704bbac9bcfb7b27ecdb484adcdb5f787901f157","pr_number":6081,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","signature_id":"96d11e5d24c2f96b51ede935923d2da2885cc5293b4b2067a83db979216fe98e","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","signature_id":"e4cd001bc94a7c7f7d880d8bd2124ba820cefb9f6079b7e26d5709adbf7dc519","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"30e7ab70c0561d3688101981f498a87ca5fd5787330e0b440c1bbb06abf887df","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"4fa3b47449004f479ac51d54be8b694777e1df03e185828cb11995c7ea0071e0","pr_number":7688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"6579f8d0a151b49461fd52fcea9ac6cb7fc7ffd497fe8e411acf99fb8274cfa3","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.6913319238900635,"anchor_occurrences":8,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,2,4,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":18,"prior_last_seen_at":"2026-10-16T10:18:04Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":8}],"full_error_samples":["time=2026-10-28T16:43:28.511Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T16:43:31.273Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T16:43:32.448Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2862080-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87\ntime=2026-10-28T16:45:32.811Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/f309630496c74d69129833a43742f2e9eec7ceb9d4411ea07b979bc73b89fd87/operationStatuses/08584084010736719502\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/cluster-rybldmoz73lrm\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2862080-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j2862080-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-28T16:47:32.817Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T16:47:34.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T16:47:35.068Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2862080-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006\ntime=2026-10-28T16:49:05.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2862080-svc/providers/Microsoft.Resources/deployments/8fa925f61f96a8233f154e3da5c65d168518bb692c424a9f42720dad4bf60006/operationStatuses/08584084008309075646\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-28T15:32:48.082Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T15:32:50.394Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T15:32:51.656Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1862528-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6\ntime=2026-10-28T15:35:22.185Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/51bb59b51713025e67da1bbb1f81509d9f0e687a1b531b2b697347d81ab333d6/operationStatuses/08584084053144753487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/cluster-5xxgmkpcjnpl2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j1862528-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j1862528-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-28T15:37:22.187Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-28T15:37:23.185Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-28T15:37:24.435Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=f330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1862528-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f\ntime=2026-10-28T15:38:54.741Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1862528-svc/providers/Microsoft.Resources/deployments/f330c62a337aea146d3097645390c7352ebc193053cfec3eadc02b161a2aba5f/operationStatuses/08584084050417029665\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-27T21:46:01.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:46:03.401Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T21:46:05.062Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0909440-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0\ntime=2026-10-27T21:48:05.587Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/ec85056f79198d17f9a4a06a9d62ec213e8ff2f13643b300963fd031a60589b0/operationStatuses/08584084693213550197\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/cluster-r5scxlmdedogk\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0909440-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j0909440-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-27T21:50:05.589Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:50:06.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-27T21:50:08.087Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=cd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0909440-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b\ntime=2026-10-27T21:51:38.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0909440-svc/providers/Microsoft.Resources/deployments/cd575f368230e7d22208e5f91e10168040367ee73eee75ae5c003a42111d906b/operationStatuses/08584084690782406556\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478468882862080","occurred_at":"2026-10-28T16:19:05Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7693,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115460075651862528","occurred_at":"2026-10-28T15:05:59Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7781,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115193188430909440","occurred_at":"2026-10-27T21:25:29Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112713075036160","occurred_at":"2026-10-27T16:05:42Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115034279682510848","occurred_at":"2026-10-27T10:54:02Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7041,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115026232805429248","occurred_at":"2026-10-27T10:22:03Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114852418565967872","occurred_at":"2026-10-26T22:51:23Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7754/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114696412376600576","occurred_at":"2026-10-26T12:31:28Z","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","pr_number":7754,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2311583e7d13fd154737ba33e5307c625f5b4961228a03e9e15f0c2f47ecff69","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":8,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,5,0,0,0,0,0,3],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T18:39:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0xf5ad11741b0\u003e: \n    resource group \"rg-candidate-5-0-lf2cjh6fwp8g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-28vlzn\" in resourcegroup=\"rg-candidate-5-0-lf2cjh6fwp8g\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n      \"name\": \"ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:57:47.524111456Z\",\n      \"endTime\": \"2026-10-28T05:21:51.687390361Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n            correlation ID: 57653937-379d-4e91-8cdc-72bb96f14f99\n            client request ID: da17b50d-815e-4f7b-b6a1-511dd538a3b6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0xf5ad11741b0\u003e: \n    resource group \"rg-candidate-5-0-lf2cjh6fwp8g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-28vlzn\" in resourcegroup=\"rg-candidate-5-0-lf2cjh6fwp8g\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n      \"name\": \"ee641f30-7b7c-4096-8a0a-8e212ff6df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:57:47.524111456Z\",\n      \"endTime\": \"2026-10-28T05:21:51.687390361Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee641f30-7b7c-4096-8a0a-8e212ff6df0d\n            correlation ID: 57653937-379d-4e91-8cdc-72bb96f14f99\n            client request ID: da17b50d-815e-4f7b-b6a1-511dd538a3b6\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2fb9767520c0\u003e: \n    resource group \"rg-candidate-4-23-8gsj9lgrgthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-x7cmb2\" in resourcegroup=\"rg-candidate-4-23-8gsj9lgrgthl\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n      \"name\": \"c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:49:53.842986291Z\",\n      \"endTime\": \"2026-10-28T05:14:02.592266097Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n            correlation ID: 4e7a26de-7852-4792-8175-683cd23ec9a8\n            client request ID: a8483675-393f-439a-880f-cb8cebaae1a8\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2fb9767520c0\u003e: \n    resource group \"rg-candidate-4-23-8gsj9lgrgthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-x7cmb2\" in resourcegroup=\"rg-candidate-4-23-8gsj9lgrgthl\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n      \"name\": \"c5e082c0-738b-4be7-9ea3-e60e834b86c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:49:53.842986291Z\",\n      \"endTime\": \"2026-10-28T05:14:02.592266097Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5e082c0-738b-4be7-9ea3-e60e834b86c8\n            correlation ID: 4e7a26de-7852-4792-8175-683cd23ec9a8\n            client request ID: a8483675-393f-439a-880f-cb8cebaae1a8\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x19fdcb2b4318\u003e: \n    resource group \"rg-candidate-4-22-b4lvcfvhwx52\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jsq2fr\" in resourcegroup=\"rg-candidate-4-22-b4lvcfvhwx52\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\",\n      \"name\": \"6f392de4-c202-4b15-9624-974c650cb629\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:51:16.927309848Z\",\n      \"endTime\": \"2026-10-28T05:15:21.671423449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n            correlation ID: 6a7c51a3-41c5-4f0f-a544-ee94e68fe93c\n            client request ID: cb15b9e0-1085-4828-b59c-20b42a78c8ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x19fdcb2b4318\u003e: \n    resource group \"rg-candidate-4-22-b4lvcfvhwx52\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jsq2fr\" in resourcegroup=\"rg-candidate-4-22-b4lvcfvhwx52\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\",\n      \"name\": \"6f392de4-c202-4b15-9624-974c650cb629\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:51:16.927309848Z\",\n      \"endTime\": \"2026-10-28T05:15:21.671423449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6f392de4-c202-4b15-9624-974c650cb629\n            correlation ID: 6a7c51a3-41c5-4f0f-a544-ee94e68fe93c\n            client request ID: cb15b9e0-1085-4828-b59c-20b42a78c8ff\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"2bb63a1891dcda7670352e5cf272ec5241fe929f6c20cde1d439939704914208","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"2ea827d729fd1f9787e245343c8d8dc70b01e2bbe151064c5ee6e34c372663ce","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"d21ae0d233481d01eee052f837a000b2cc41b93390d9f31d585d94518f7b055e","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"3a71b87c724dd30609861eaf297fd7c712b6b149707e839f6721959f2a1a471d","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"a6a6d7ef95b9bce2823ee44c5133f92a6c026911f7cba3ed9bfc9fb6134bd9b7","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"f6273eed30538960aa15eba547ebb2f777f14cefb915530b47ab177adf25601c","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","signature_id":"7d48d0783a7d6fa8287cfe8694ee010e0fe2aa87656f0647076142aac123bae1","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","signature_id":"e89a1c1574690bc16c1102c73369b4e5d51aa6d1b3d8433e36b4576eb66adc6f","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4799154334038056,"anchor_occurrences":7,"after_last_push_count":2,"also_in":["INT"],"trend_counts":[1,0,2,0,0,0,2,3],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-23T12:42:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0x740ee223cf8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-dmmfjzfldpm8\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\",\n      \"name\": \"e51bb1e4-4996-454f-a81d-9354613a75ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:36:46.639467207Z\",\n      \"endTime\": \"2026-10-28T19:55:53.152282953Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-8z842-x96hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-8z842-tlsx7: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-dmmfjzfldpm8--managed/cilium-cl-cilium-np-8z842-tlsx7-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-dmmfjzfldpm8--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-8z842-tlsx7-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1d2f4929-515e-4e8d-8580-836e819f8d50' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n            correlation ID: 4963ae88-5109-4671-873f-f28a9deccdac\n            client request ID: 38a19ead-d2a2-4130-9e04-1364166950d2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0x740ee223cf8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-dmmfjzfldpm8\" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\",\n      \"name\": \"e51bb1e4-4996-454f-a81d-9354613a75ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:36:46.639467207Z\",\n      \"endTime\": \"2026-10-28T19:55:53.152282953Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-8z842-x96hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-8z842-tlsx7: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-dmmfjzfldpm8--managed/cilium-cl-cilium-np-8z842-tlsx7-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-dmmfjzfldpm8--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-8z842-tlsx7-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1d2f4929-515e-4e8d-8580-836e819f8d50' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e51bb1e4-4996-454f-a81d-9354613a75ee\n            correlation ID: 4963ae88-5109-4671-873f-f28a9deccdac\n            client request ID: 38a19ead-d2a2-4130-9e04-1364166950d2\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x335ced2f1b40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfwkzrbn6vbm\" to finish creating: GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n      \"name\": \"77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T16:28:12.935779531Z\",\n      \"endTime\": \"2026-10-28T16:47:20.605052936Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-kfwkzrbn6vbm--managed/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfwkzrbn6vbm--managed/providers/Microsoft.Compute/virtualMachines/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n            correlation ID: c83d59d4-8297-41d1-87c1-fbe0352a6163\n            client request ID: 4dcc2ce6-6521-4d7f-b86f-f0309a80ecee\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x335ced2f1b40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfwkzrbn6vbm\" to finish creating: GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n      \"name\": \"77628145-9ee8-4d12-9dd2-5dced29b5ecb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T16:28:12.935779531Z\",\n      \"endTime\": \"2026-10-28T16:47:20.605052936Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7: Failed: virtualmachine failed to create or update. err: failed to get existing resource fips-enabled-kfwkzrbn6vbm--managed/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfwkzrbn6vbm--managed/providers/Microsoft.Compute/virtualMachines/j6k4t1t6z4j2k5z-np-1-5nm58-fsnz7\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https:... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4206336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77628145-9ee8-4d12-9dd2-5dced29b5ecb\n            correlation ID: c83d59d4-8297-41d1-87c1-fbe0352a6163\n            client request ID: 4dcc2ce6-6521-4d7f-b86f-f0309a80ecee\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ecbca3c5be0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-p9l8lv6w4lzt\" to finish creating: GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n      \"name\": \"0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T06:54:59.774467587Z\",\n      \"endTime\": \"2026-10-28T07:14:00.6647054Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mxkn8-rr6bs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mxkn8-rr6bs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-mxkn8-c8z72: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-p9l8lv6w4lzt--managed/v20260901-np-1-mxkn8-c8z72-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-mxkn8-c8z72-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n            correlation ID: 5492e0f1-946f-49bb-8474-ef7f078e1f6f\n            client request ID: 12bebe6d-b861-48ec-942f-b913321fe64f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ecbca3c5be0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-p9l8lv6w4lzt\" to finish creating: GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n      \"name\": \"0d4205c1-5db7-479b-92dc-43f88aa7a88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T06:54:59.774467587Z\",\n      \"endTime\": \"2026-10-28T07:14:00.6647054Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mxkn8-rr6bs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mxkn8-rr6bs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-mxkn8-c8z72: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-p9l8lv6w4lzt--managed/v20260901-np-1-mxkn8-c8z72-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-p9l8lv6w4lzt--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-mxkn8-c8z72-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5142016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0d4205c1-5db7-479b-92dc-43f88aa7a88d\n            correlation ID: 5492e0f1-946f-49bb-8474-ef7f078e1f6f\n            client request ID: 12bebe6d-b861-48ec-942f-b913321fe64f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512320468717568","occurred_at":"2026-10-28T18:33:36Z","signature_id":"68ad6dfac63b490246484162a2cfeb7d64d4970cbbabdb4db8652e41011fa267","pr_number":7764,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462359144206336","occurred_at":"2026-10-28T15:15:04Z","signature_id":"e95a85e5040a99543bdcfc0978c5a8dd4686775675fab4d33590a63d57268ab9","pr_number":7704,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7795/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115320649005142016","occurred_at":"2026-10-28T05:51:58Z","signature_id":"482b9f3179c10e36672118678ca7d0b64b36a887d80217ac5fb7f60bda8d3d24","pr_number":7795,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216026894340096","occurred_at":"2026-10-27T22:56:14Z","signature_id":"4d308ae304f8d76d5c0a297940b13882945a80c9d3b647dbda29ddcdd9f64ba2","pr_number":7769,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115163835793936384","occurred_at":"2026-10-27T19:28:50Z","signature_id":"6933fc9bb0db831be293275832c4bad7fce83abb554cf2ccb7be65585cfd3714","pr_number":5244,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","signature_id":"2105a054cb380d7c595411f3110d328751601f1b100d33affce72e45f10a0150","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579232666849280","occurred_at":"2026-10-23T10:32:12Z","signature_id":"9813e169df513a9dded1ccb5b4bf7b1a057e15d6c3c68da4e0c1eb71f143dc1c","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4799154334038056,"anchor_occurrences":7,"after_last_push_count":3,"trend_counts":[0,1,4,1,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-25T11:17:06Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":7}],"full_error_samples":["time=2026-10-25T11:53:02.786Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-25T11:53:04.190Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-25T11:53:04.190Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:53:05.198Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8850688-svc\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=3m5s\ntime=2026-10-25T11:56:09.852Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:10.416Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.207Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-25T11:56:11.207Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.237Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-25T11:56:11.450Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.510Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8850688-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-25T11:56:11.971Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j8850688-svc\ntime=2026-10-25T12:11:14.972Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-24T03:19:41.940Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T03:19:44.124Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T03:19:44.124Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j0729472-svc\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m6s\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.375Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-24T03:21:52.375Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.438Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-24T03:21:52.730Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.857Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j0729472-svc\ntime=2026-10-24T03:36:56.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-24T00:11:13.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:11:15.609Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m4s\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.894Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-24T00:13:21.097Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.168Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:28:24.267Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114315309748850688","occurred_at":"2026-10-25T11:17:06Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7597,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113821515370729472","occurred_at":"2026-10-24T02:34:56Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113774695802212352","occurred_at":"2026-10-23T23:28:54Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7120,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113750398169780224","occurred_at":"2026-10-23T21:52:21Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7120,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683432016973824","occurred_at":"2026-10-23T17:26:03Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113129581849350144","occurred_at":"2026-10-22T04:45:26Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7041,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.4799154334038056,"anchor_occurrences":7,"after_last_push_count":3,"trend_counts":[2,3,2,2,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-24T16:12:09Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":7}],"full_error_samples":["time=2026-10-24T16:56:57.345Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T16:56:58.695Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T16:56:58.695Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j5468416-svc\ntime=2026-10-24T17:12:01.792Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-24T12:42:08.055Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T12:42:09.448Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T12:42:09.448Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j1222656-svc\ntime=2026-10-24T12:57:12.942Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-23T10:04:28.865Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4942976-svc\ntime=2026-10-23T10:19:36.095Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114027174305468416","occurred_at":"2026-10-24T16:12:09Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113964216481222656","occurred_at":"2026-10-24T12:01:59Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7741,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7711/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113559441314942976","occurred_at":"2026-10-23T09:13:33Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7711,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113434930414358528","occurred_at":"2026-10-23T00:58:47Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7705,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113385464588144640","occurred_at":"2026-10-22T21:42:14Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113203785768636416","occurred_at":"2026-10-22T09:40:18Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7665,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113142261167427584","occurred_at":"2026-10-22T05:35:49Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7653,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":7,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,6,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-12","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-q5hjsn\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18f75457b1e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-q5hjsn: failed waiting for cluster=\"cluster-zstream-4-20-q5hjsn\" in resourcegroup=\"rg-zstream-upgrade-4-20-jb8s5lbcxszk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n      \"name\": \"5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:30.073248883Z\",\n      \"endTime\": \"2026-10-25T11:30:35.871032188Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.42 is Partial (want Completed), started 23m47s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n            correlation ID: cf35d619-4e23-43fc-8aeb-ee92df9074e6\n            client request ID: 68e2030f-31dd-4592-871e-2e8df326a35b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-q5hjsn\" with version 4.20.42 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18f75457b1e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-q5hjsn: failed waiting for cluster=\"cluster-zstream-4-20-q5hjsn\" in resourcegroup=\"rg-zstream-upgrade-4-20-jb8s5lbcxszk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n      \"name\": \"5400fbc4-9a5a-4a07-8b68-53299b9929f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:30.073248883Z\",\n      \"endTime\": \"2026-10-25T11:30:35.871032188Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.42 is Partial (want Completed), started 23m47s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5400fbc4-9a5a-4a07-8b68-53299b9929f5\n            correlation ID: cf35d619-4e23-43fc-8aeb-ee92df9074e6\n            client request ID: 68e2030f-31dd-4592-871e-2e8df326a35b\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x134cca4de620\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-smqlw28gpmfk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n      \"name\": \"d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:59.115997025Z\",\n      \"endTime\": \"2026-10-25T11:31:07.16366343Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m52s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n            correlation ID: 9f778ec9-bb64-4e1c-9a2f-a32317e4bb4a\n            client request ID: bfadd86d-0c2c-4751-a28f-5658e7a124e6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x134cca4de620\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-smqlw28gpmfk\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n      \"name\": \"d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:59.115997025Z\",\n      \"endTime\": \"2026-10-25T11:31:07.16366343Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m52s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4d8ca67-31e0-4a7f-b038-d6a3a610c1f2\n            correlation ID: 9f778ec9-bb64-4e1c-9a2f-a32317e4bb4a\n            client request ID: bfadd86d-0c2c-4751-a28f-5658e7a124e6\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b59459fc100\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-t7spx49mgg2h\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n      \"name\": \"fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:26.130044072Z\",\n      \"endTime\": \"2026-10-25T11:30:36.059795656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 22m37s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n            correlation ID: 1127ed99-2d51-4906-b522-5ba778a6c853\n            client request ID: eb7f7962-a820-4a0a-acaa-426a2d0c04e3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b59459fc100\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-t7spx49mgg2h\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n      \"name\": \"fde9935e-6ef0-4518-bbc5-60b5d4f16d38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:26.130044072Z\",\n      \"endTime\": \"2026-10-25T11:30:36.059795656Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 22m37s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fde9935e-6ef0-4518-bbc5-60b5d4f16d38\n            correlation ID: 1127ed99-2d51-4906-b522-5ba778a6c853\n            client request ID: eb7f7962-a820-4a0a-acaa-426a2d0c04e3\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"23e2eb050a4416384db3752c651a9d3db7b8c538cc0bf856ae3091acdd9ca6d1","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"6876faa6b24b00dfa66f80f783d33f416f7acda715020a397c52cb5732457de1","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"8ad851bf221457dbbdf65329df79357aee7c2ceca689a609b15097ec5d216d30","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"c2246a6f55fb01eedd384fdcab8c1f79fabf97dc38129537b05dd14067f93470","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"d9a5d39edfcea9c2671c10762371677ac7311bb7caba7fdb15a801d02fd90b80","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"f763f825d92522261c47dffbfc1adbb9088da2a539dfe81ca7c6af2d8a1d41f4","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"e1da39dbfa2686ab72adc9980862457714384b0d27434b03e987e157712cc1b9","pr_number":7657,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0570824524312896,"anchor_occurrences":5,"after_last_push_count":3,"also_in":["INT","PROD"],"trend_counts":[0,3,0,0,0,0,0,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-22T20:56:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-krfsmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34ab4bc30760\u003e: \n    DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x34ab4c426760\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-krfsmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34ab4bc30760\u003e: \n    DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x34ab4c426760\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-hlljf.apps.aro.c5q3m7v0m2r2b6f.va75.j8974848.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-5pj7h6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2590ea2143e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2590e99482e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-5pj7h6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2590ea2143e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2590e99482e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dsbx7.apps.aro.m8c7u6n4b6i4x7c.8inl.j9099392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-8s2f2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30cafd6d0a20\u003e: \n    DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30cafd44e600\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-8s2f2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30cafd6d0a20\u003e: \n    DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30cafd44e600\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","signature_id":"a9ef1c5a61aa310aaaf853ef996be4e9f1f883da0ee59e7e69e9b3a3e5b3cd68","pr_number":7786,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7797/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115341484919099392","occurred_at":"2026-10-28T07:14:45Z","signature_id":"59c18568aacd2ba62746c2c266f8fd637816b7e9ca52d7fa2eca6c6ee1162774","pr_number":7797,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113373941157335040","occurred_at":"2026-10-22T20:56:26Z","signature_id":"e0dcd02397ab4d23ba24e7c57448f2841b01c0fa54fb3bb9faad619af0f10aa6","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113151915083698176","occurred_at":"2026-10-22T06:14:11Z","signature_id":"dc531b14207c6f576efeb1d0b689a61a1f6e47c8a24d2b02b6c10b19d54c8208","pr_number":7672,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113130790136057856","occurred_at":"2026-10-22T04:50:15Z","signature_id":"8d094efb455f76798c9e8a4cce31f30c0ce80c0c9a0fd1d2e952804b6d1dc9b7","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0570824524312896,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,1,1,0,2,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-25T09:55:02Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T01:28:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j5915264-svc\", severity=\"warning\"\n  Description: Credential refresher expiration metric is missing for service cluster ci01-j5915264-svc.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T21:37:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j3842944-svc\", severity=\"warning\"\n  Description: Credential refresher expiration metric is missing for service cluster ci01-j3842944-svc.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T18:18:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j4499328-svc\", severity=\"warning\"\n  Description: Credential refresher expiration metric is missing for service cluster ci01-j4499328-svc."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115235750155915264","occurred_at":"2026-10-28T00:14:36Z","signature_id":"dde57ce0fcd4d68140d4c1efab2ca64984da83e3c3b4ce209b60569f896e7311","pr_number":6081,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174942373842944","occurred_at":"2026-10-27T20:12:58Z","signature_id":"c48979391c5633bf7cf1f49fba3381dc030e10cbe77d463ae9e3618fb8f7821a","pr_number":6081,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127036694499328","occurred_at":"2026-10-27T17:02:37Z","signature_id":"bdaf4cc6230c0864f55513a6432317c85b8bb17698744582bcc8c62c19c0698b","pr_number":6081,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294657641877504","occurred_at":"2026-10-25T09:55:02Z","signature_id":"2a2f274dff4f6f6fc65910529a8fbb12b08a54cbcdfc227ca6f705b9a23049d4","pr_number":6081,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797768924368896","occurred_at":"2026-10-24T01:00:13Z","signature_id":"6ef254c858e87304545e34d73618328753d7ee1ecc5a887748fef0be19e5e856","pr_number":6081,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.0570824524312896,"anchor_occurrences":5,"after_last_push_count":3,"trend_counts":[1,2,1,1,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-24T08:29:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:42:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T14:43:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8622848-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8622848/providers/microsoft.monitor/accounts/services-j8622848\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-xxzkn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-24T10:10:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-24T10:10:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-23T12:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-23T12:56:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115069760348622848","occurred_at":"2026-10-27T13:15:01Z","signature_id":"7442bf20121518a5319998062acd103d25b97d6fedb3ac8a1e6323c1a65edc4c","pr_number":7615,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"7de7d944b459e50dfdd29dedf94b6727c8e84d62f3395cbb57514c85a8e7a176","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"7467c8b82e1de4741264fd7a83a7e3d54e11cebdb1e11c074699979a6d8ec2ff","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113283987379064832","occurred_at":"2026-10-22T14:58:49Z","signature_id":"6610592e09db876ec6c37c80a3d1988cc8a66321429ef0c56dab5208368f0124","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254479959691264","occurred_at":"2026-10-22T13:01:44Z","signature_id":"5f46b56ac477343a6a11acbdfacf4c4c593c390c995318b85beadcc73d4ed5ba","pr_number":7133,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":5,"after_last_push_count":3,"also_in":["INT"],"trend_counts":[1,4,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":22,"prior_last_seen_at":"2026-10-23T02:36:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n      \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n      \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n            correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n            client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n      \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n      \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n            correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n            client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202fbf30a20\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster=\"cluster-candidate-4-23-q8dfp4\" in resourcegroup=\"rg-candidate-4-23-h5dw5xj4gsz5\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\",\n      \"name\": \"1daf02db-3317-4617-a13e-d6a72278331e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T22:42:59.591011904Z\",\n      \"endTime\": \"2026-10-22T23:12:07.791955181Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n            correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853\n            client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x202fbf30a20\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster=\"cluster-candidate-4-23-q8dfp4\" in resourcegroup=\"rg-candidate-4-23-h5dw5xj4gsz5\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\",\n      \"name\": \"1daf02db-3317-4617-a13e-d6a72278331e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T22:42:59.591011904Z\",\n      \"endTime\": \"2026-10-22T23:12:07.791955181Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e\n            correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853\n            client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a093dfe4480\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster=\"cluster-candidate-5-0-9zpb54\" in resourcegroup=\"rg-candidate-5-0-7v6qdnlhwvvb\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n      \"name\": \"23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T22:43:09.924601415Z\",\n      \"endTime\": \"2026-10-22T23:12:18.105452223Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n            correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691\n            client request ID: 5ec430fe-78af-4487-8945-3607302f78b7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a093dfe4480\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster=\"cluster-candidate-5-0-9zpb54\" in resourcegroup=\"rg-candidate-5-0-7v6qdnlhwvvb\" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n      \"name\": \"23ba9cb2-0511-4188-a063-cf6d4e7f1fde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T22:43:09.924601415Z\",\n      \"endTime\": \"2026-10-22T23:12:18.105452223Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde\n            correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691\n            client request ID: 5ec430fe-78af-4487-8945-3607302f78b7\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"a0d813b5252e54924e130bf68e119ec22802307c2b8775c3330243b5fac10ac1","pr_number":7687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","signature_id":"200dd7d90d12350bb3d5692f2441c7f6fbacb47c87dbb8b771b20f5be28bb4f1","pr_number":7548,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113387154783604736","occurred_at":"2026-10-22T21:48:57Z","signature_id":"58649267055e4c23aa219ef1853e3253b93ce6e21498804b8d24cf5a47de3412","pr_number":7548,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","signature_id":"a9050521680fae4d294a8aeac103dcc74b0cc4f6524dabc513da8d4e5d2a6777","pr_number":7653,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113256572250492928","occurred_at":"2026-10-22T13:10:03Z","signature_id":"a5b2daf3dd240c9a4da33b2d49221615c7d48c11644f231e38b5041bfd0487d6","pr_number":7633,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":4,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[2,1,1,1,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-24T00:49:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19ee49ef83c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwr8wdxmg7md\" to finish creating: GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"name\": \"e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:09:15.328141673Z\",\n      \"endTime\": \"2026-10-27T16:28:23.766899649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n            correlation ID: 963eb402-1e02-4815-b759-a4a687cc24af\n            client request ID: 393f46a9-acc4-4c32-9f2a-527f2574f38e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19ee49ef83c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwr8wdxmg7md\" to finish creating: GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"name\": \"e7d2379c-44d1-408b-924b-bb3e4a41c31e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:09:15.328141673Z\",\n      \"endTime\": \"2026-10-27T16:28:23.766899649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o0j6i6a1b6t0q5h-np-1-9klrv-7r7qm: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9721728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7d2379c-44d1-408b-924b-bb3e4a41c31e\n            correlation ID: 963eb402-1e02-4815-b759-a4a687cc24af\n            client request ID: 393f46a9-acc4-4c32-9f2a-527f2574f38e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xda86742c750\u003e: \n    VerifyNodesReady failed: there are not ready nodes: cilium-cluster-cilium-np-kph42-vfrqk (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-24T02:36:22Z)\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwbfcwb579g7\" to finish creating: GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n      \"name\": \"40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T02:18:41.838217414Z\",\n      \"endTime\": \"2026-10-24T02:37:46.632619034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n            correlation ID: 260f62ef-02cf-4e9c-83e2-0f2095de0a22\n            client request ID: 0ea1151f-ae49-4a4c-83b3-a2443bb08376\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xda86742c750\u003e: \n    VerifyNodesReady failed: there are not ready nodes: cilium-cluster-cilium-np-kph42-vfrqk (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-24T02:36:22Z)\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwbfcwb579g7\" to finish creating: GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n      \"name\": \"40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T02:18:41.838217414Z\",\n      \"endTime\": \"2026-10-24T02:37:46.632619034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n            correlation ID: 260f62ef-02cf-4e9c-83e2-0f2095de0a22\n            client request ID: 0ea1151f-ae49-4a4c-83b3-a2443bb08376\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n      \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n      \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n            correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n            client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n      \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n      \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n            correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n            client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115098874459721728","occurred_at":"2026-10-27T15:10:42Z","signature_id":"29d10a662a78238c732f153094e693885b24ff3acbe770818665b4faa0fcd90a","pr_number":7700,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113794936213082112","occurred_at":"2026-10-24T00:49:19Z","signature_id":"d0567201314e2f51df5880f0f1f4bfed0b2ec9921b0de43f8e38feddc4717086","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","signature_id":"d9cde6d3be6b80b67637a1674fb67e65e7c5c386976a816549974fb5fd2ebc69","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254320458698752","occurred_at":"2026-10-22T13:01:04Z","signature_id":"f11e42bee52762323ac7f2a29ccc1f770e75b1825e14bc3e92057295a1ae7fb3","pr_number":7548,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,1,1,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-23T02:36:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c827a968340\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9k6kks4sqmhh\" to finish creating: GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n      \"name\": \"e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:17:13.271950373Z\",\n      \"endTime\": \"2026-10-28T17:36:19.733049446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n            correlation ID: e5b7a290-af02-4ff7-9f0a-851a1b652878\n            client request ID: a046022e-1fb3-43ea-9318-d95c8087e3da\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c827a968340\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9k6kks4sqmhh\" to finish creating: GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n      \"name\": \"e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:17:13.271950373Z\",\n      \"endTime\": \"2026-10-28T17:36:19.733049446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9k6kks4sqmhh--managed/providers/Microsoft.Compute/virtualMachines/x5b5y8h0f8t5k2t-autoscale-noaz-b47sx-574cm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5392000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e7e6b86c-a0ff-4c35-94e7-3aa8b285e19e\n            correlation ID: e5b7a290-af02-4ff7-9f0a-851a1b652878\n            client request ID: a046022e-1fb3-43ea-9318-d95c8087e3da\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c3a9a99f660\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hqzrkb5qdbkx\" to finish creating: GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"name\": \"ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:11:38.810985949Z\",\n      \"endTime\": \"2026-10-27T20:30:42.147810175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n            correlation ID: 20b2a10f-dd2b-4a62-8cff-21fa59649da0\n            client request ID: a877eec2-04c5-44bd-9434-61383daeb4f5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c3a9a99f660\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hqzrkb5qdbkx\" to finish creating: GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"name\": \"ff22fd81-847d-4bcb-b22f-4250d9071095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T20:11:38.810985949Z\",\n      \"endTime\": \"2026-10-27T20:30:42.147810175Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-hqzrkb5qdbkx--managed/providers/Microsoft.Compute/virtualMachines/m7z0h7l7n3n4n3s-autoscale-noaz-hfvwg-htn8f to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1666560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff22fd81-847d-4bcb-b22f-4250d9071095\n            correlation ID: 20b2a10f-dd2b-4a62-8cff-21fa59649da0\n            client request ID: a877eec2-04c5-44bd-9434-61383daeb4f5\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa75b49b000\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c7rz5j8lqwzr\" to finish creating: GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"name\": \"116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:04:30.855869275Z\",\n      \"endTime\": \"2026-10-26T16:23:34.187561349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n            correlation ID: 0413d2ef-0809-4a88-a400-52222573fc14\n            client request ID: 2d020f95-fbad-4853-8497-2a3822d5b948\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa75b49b000\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c7rz5j8lqwzr\" to finish creating: GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"name\": \"116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:04:30.855869275Z\",\n      \"endTime\": \"2026-10-26T16:23:34.187561349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n            correlation ID: 0413d2ef-0809-4a88-a400-52222573fc14\n            client request ID: 2d020f95-fbad-4853-8497-2a3822d5b948\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115476335995392000","occurred_at":"2026-10-28T16:10:35Z","signature_id":"2cb0078a8e25d7023016829d59625bdf6e299ff4ac00dcfbbeb79a69cbfa76b3","pr_number":7809,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115149099081666560","occurred_at":"2026-10-27T17:40:53Z","signature_id":"59414b931b98b1d5d286decc70f822294d05cabdb5a1afce423cbe142550f9fb","pr_number":7781,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","signature_id":"71b4b6dc383e5a878ae2899edaf985dd95ce3a388ec2b470b38589174c5894ad","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"361d3eae8393f6eef155ba76cade65d4102abb09f4a193a37d5f4161c2bf94bf","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[1,1,1,0,0,0,1,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-23T15:56:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc83633402b8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc83630bcee0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xc8362e9fb78\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xc83633b5610\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc83633402b8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc83630bcee0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xc8362e9fb78\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xc83633b5610\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xbc827a50558\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xbc827a00340\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xbc827b00060\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xbc826b52230\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:183]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xbc827a50558\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xbc827a00340\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0xbc827b00060\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0xbc826b52230\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.20:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","signature_id":"636358cf2ad9136d60ba8fa1f4f46495eeb3f3f4d0f9eaa4658ff7f22e6bdd37","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115181315694268416","occurred_at":"2026-10-27T20:38:18Z","signature_id":"b6c5bc582704a39e441546c12f0a5da8432ee3cd1585874fdfcf70c165b8f04c","pr_number":7786,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","signature_id":"4b06ceefd21c3c876e41349b56654277c2324b6708e3809cfe51f45fabf75683","pr_number":7073,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113138915056553984","occurred_at":"2026-10-22T05:22:32Z","signature_id":"1a3274025bba0a36059024cc04b88e2285c5367a4778ef67fe03503d1eb13a3e","pr_number":7679,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.8456659619450317,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["INT"],"trend_counts":[0,0,2,2,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-24T13:06:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2617933fd6e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2617933fd6e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc346b690c20\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc346b690c20\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1355449513d0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1355449513d0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113980401566355456","occurred_at":"2026-10-24T13:06:18Z","signature_id":"cdd4a365bd1f8f5f93c3078adedb43cf9bd9a254480851d78e85c02f3b74fcb3","pr_number":7741,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113895350061240320","occurred_at":"2026-10-24T07:28:20Z","signature_id":"c4f2bd800b94fe1f64cf0dcf512f1253dfba262d21febcfcbd9981b5b70eed50","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","signature_id":"24816c79684a0f74c02483d2e3b9365bf73a0c2a88310b47fae43e1866fb1d1a","pr_number":6770,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"3582f51ef43ab6cebf49738a8faa79e10138b0aa9aed91bb247ce640202ba0e7","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":4,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-24T12:30:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2v886t\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2ea218b7e820\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2ea218b7e780\u003e{\n            observation: \u003c*errors.errorString | 0x2ea218d2a9b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2ea218b7e760\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2v886t\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2ea218b7e820\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2ea218b7e780\u003e{\n            observation: \u003c*errors.errorString | 0x2ea218d2a9b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2ea218b7e760\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-9vhc67\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2f35f4c5c1a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2f35f4c5c120\u003e{\n            observation: \u003c*errors.errorString | 0x2f35f6f1e830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2f35f4c5c0c0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-9vhc67\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2f35f4c5c1a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2f35f4c5c120\u003e{\n            observation: \u003c*errors.errorString | 0x2f35f6f1e830\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2f35f4c5c0c0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-84tw8j\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x347780605600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x3477806055c0\u003e{\n            observation: \u003c*errors.errorString | 0x34778111e160\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x3477806055a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-84tw8j\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x347780605600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x3477806055c0\u003e{\n            observation: \u003c*errors.errorString | 0x34778111e160\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x3477806055a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","signature_id":"936e85aac84b68ba9d7b02cd2e8d81f0c398e3411296a8498b63a2315e4cba81","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"91906ee05ec48da05bfce86c04cd6b54b0c64ff95e2a58a358219124f81c3325","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","signature_id":"1c556c68dc0e81b592bd96ea684cc1bf9b6af1fb39f1b4904e1224a3d612c036","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","signature_id":"9b0fa023f70c43f9bd584cf2b92c6a19d34fe34c5b3a361cf4f12f49592982d4","pr_number":7734,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":4,"after_last_push_count":6,"also_in":["PROD"],"trend_counts":[0,3,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-23T11:17:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n    resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n      \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n      \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n            correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n            client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n    resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n      \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n      \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n            correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n            client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a775c43e180\u003e: \n    resource group \"complex-cilium-kv-8hx8sgkq9px6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8hx8sgkq9px6\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n      \"name\": \"28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:21:07.679040667Z\",\n      \"endTime\": \"2026-10-22T14:45:09.479893276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n            correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946\n            client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a775c43e180\u003e: \n    resource group \"complex-cilium-kv-8hx8sgkq9px6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8hx8sgkq9px6\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n      \"name\": \"28cf5da8-4bf8-49ef-8a24-44a6ab872d73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:21:07.679040667Z\",\n      \"endTime\": \"2026-10-22T14:45:09.479893276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73\n            correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946\n            client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x239068445020\u003e: \n    resource group \"cni-cilium-w6t5xjcmq5lh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-w6t5xjcmq5lh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n      \"name\": \"47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:19:53.04024209Z\",\n      \"endTime\": \"2026-10-22T14:43:59.86158375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n            correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4\n            client request ID: db330e23-e554-4902-a60c-dc62e09132f3\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x239068445020\u003e: \n    resource group \"cni-cilium-w6t5xjcmq5lh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-w6t5xjcmq5lh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n      \"name\": \"47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:19:53.04024209Z\",\n      \"endTime\": \"2026-10-22T14:43:59.86158375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c\n            correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4\n            client request ID: db330e23-e554-4902-a60c-dc62e09132f3\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"2623387444ac60aad1c81d411318d579ab144dfc53c580add9b2105a4d1cf697","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"21381fe06efc73c3442517b08384543c00261e1938f3ca66dc65ef940b6459da","pr_number":7686,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"78240a56007cbdb142d1fe54126202e27b95da39d24a277343ee78126e9f1890","pr_number":7686,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"aeb431818b5c6b442b52161098d914e998eb3001d4b357dac24274924a88a237","pr_number":7686,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-13T15:32:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":2},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert collection is complete","occurrences":2}],"full_error_samples":["alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j2704640' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7793,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7793,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-cv7tq","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":3,"after_last_push_count":1,"also_in":["INT","PROD"],"trend_counts":[0,0,1,1,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-24T17:33:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:545]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x303527456700\u003e: \n    DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3035274566c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:545]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x303527456700\u003e: \n    DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3035274566c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cv7tq.apps.aro.ingress-cert-le.bfz4.j3936384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11989c84120\u003e: \n    DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x11989c840a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11989c84120\u003e: \n    DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x11989c840a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n    DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n    DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115163835793936384","occurred_at":"2026-10-27T19:28:50Z","signature_id":"d07a57d85a5fbf19cecde1119f1ea8aa1c9e7ad4e3e554a72b71831011228ce2","pr_number":5244,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114047539169726464","occurred_at":"2026-10-24T17:33:05Z","signature_id":"685d28f37bdd0fa7fcacd74adb60aa052fa898956388730f22231c7fb6a1c8ba","pr_number":7730,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"3e24e82e343cd7508ed60a7ec8730ea060e11e44c4fe22d5afc11ad0663e5eaa","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[1,1,0,1,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-24T10:27:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc2b6fc92f40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4pl8j4mzdr2l\" to finish creating: GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n      \"name\": \"67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T18:04:23.346337134Z\",\n      \"endTime\": \"2026-10-28T18:23:31.470906003Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-4pl8j4mzdr2l--managed/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-4pl8j4mzdr2l--managed/providers/Microsoft.Network/networkInterfaces/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f0916b6b-4fcf-4b8b-b7f8-5a66898b1190' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n            correlation ID: 7d414729-3b90-4c4b-9628-b2993ebd631c\n            client request ID: 0c6fdc27-946b-4c40-9ac6-c7629afa8ed5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc2b6fc92f40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4pl8j4mzdr2l\" to finish creating: GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n      \"name\": \"67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T18:04:23.346337134Z\",\n      \"endTime\": \"2026-10-28T18:23:31.470906003Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-4pl8j4mzdr2l--managed/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-4pl8j4mzdr2l--managed/providers/Microsoft.Network/networkInterfaces/v2e1t9h4b7t5u0a-np-1-cd7br-b9qvn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f0916b6b-4fcf-4b8b-b7f8-5a66898b1190' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0412544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67bb2317-9c7e-4d3b-8e3c-0ad00e76a125\n            correlation ID: 7d414729-3b90-4c4b-9628-b2993ebd631c\n            client request ID: 0c6fdc27-946b-4c40-9ac6-c7629afa8ed5\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x232a21da8bc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5cnqwcb679kx\" to finish creating: GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n      \"name\": \"eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T11:30:35.210037853Z\",\n      \"endTime\": \"2026-10-24T11:49:35.657573237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Compute/virtualMachines/y3p5h5t6y6r0a2w-np-1-656n9-dj76m to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-5cnqwcb679kx--managed/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Network/networkInterfaces/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n            correlation ID: 1204c613-1fa2-4780-b8d4-cf6423a998a8\n            client request ID: 70ebbcc0-cca4-424d-8160-3551b5e8f506\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x232a21da8bc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5cnqwcb679kx\" to finish creating: GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n      \"name\": \"eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T11:30:35.210037853Z\",\n      \"endTime\": \"2026-10-24T11:49:35.657573237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Compute/virtualMachines/y3p5h5t6y6r0a2w-np-1-656n9-dj76m to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-5cnqwcb679kx--managed/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Network/networkInterfaces/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n            correlation ID: 1204c613-1fa2-4780-b8d4-cf6423a998a8\n            client request ID: 70ebbcc0-cca4-424d-8160-3551b5e8f506\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e7f906828a0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hlczr5rhlkx8\" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n      \"name\": \"2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T06:07:20.975549407Z\",\n      \"endTime\": \"2026-10-22T06:26:22.595569254Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n            correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e\n            client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e7f906828a0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hlczr5rhlkx8\" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n      \"name\": \"2d4aa15e-aedb-44e3-87cf-763478cb8fc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T06:07:20.975549407Z\",\n      \"endTime\": \"2026-10-22T06:26:22.595569254Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1\n            correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e\n            client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7811/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115488597250412544","occurred_at":"2026-10-28T16:59:20Z","signature_id":"54271ff2159587172376438d500776a9f861603b592547af2dce7b7bf4770eaa","pr_number":7811,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113940526687326208","occurred_at":"2026-10-24T10:27:51Z","signature_id":"0f394417741593ba503ac249fa660e68b45000e72808dbd8475c79f316f0f9d2","pr_number":7514,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113135233485246464","occurred_at":"2026-10-22T05:07:54Z","signature_id":"0261483674c89aeb066aafc37d75b4ad9859cf5cd7d723600d46bef0f3934bdd","pr_number":7592,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["INT","PROD"],"trend_counts":[1,0,1,0,1,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-25T09:44:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26ee15ea6540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lz9smvq6kbwg\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"name\": \"ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:28:59.117849892Z\",\n      \"endTime\": \"2026-10-27T23:48:07.430527402Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n            correlation ID: e016f547-ae67-4080-be2a-aa90f089b488\n            client request ID: c52a8689-3c38-4f1f-a275-e48620136ff8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26ee15ea6540\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lz9smvq6kbwg\" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"name\": \"ea7a0229-5912-411c-9d71-ed6121f1fa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T23:28:59.117849892Z\",\n      \"endTime\": \"2026-10-27T23:48:07.430527402Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-9rrd8 to exist\\nMachine l6k1q7q7h9e3k0z-np-1-thnn4-w82sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-lz9smvq6kbwg--managed/providers/Microsoft.Compute/virtualMachines/l6k1q7q7h9e3k0z-np-1-thnn4-w82sd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ea7a0229-5912-411c-9d71-ed6121f1fa93\n            correlation ID: e016f547-ae67-4080-be2a-aa90f089b488\n            client request ID: c52a8689-3c38-4f1f-a275-e48620136ff8\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a5fafb1fdc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dhxpqc2757nk\" to finish creating: GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n      \"name\": \"995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:09:27.17231924Z\",\n      \"endTime\": \"2026-10-25T11:28:33.403436193Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n            correlation ID: d01faed1-7c94-4749-8892-78ade8753a54\n            client request ID: 5d2be0a3-7126-4924-b3ac-5c1c229ea51d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a5fafb1fdc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dhxpqc2757nk\" to finish creating: GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n      \"name\": \"995ad93d-44a1-4aae-9f3c-cef4914b4415\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:09:27.17231924Z\",\n      \"endTime\": \"2026-10-25T11:28:33.403436193Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-kjbw8 to exist\\nMachine w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dhxpqc2757nk--managed/providers/Microsoft.Compute/virtualMachines/w1u6b4w7n6r4n6j-np-1-tvg6z-njbxt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9438720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/995ad93d-44a1-4aae-9f3c-cef4914b4415\n            correlation ID: d01faed1-7c94-4749-8892-78ade8753a54\n            client request ID: 5d2be0a3-7126-4924-b3ac-5c1c229ea51d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n      \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n      \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n            correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n            client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n      \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n      \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n            correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n            client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","signature_id":"0d6baa6c9c1c38b46fb24aaf5c600e2b1a68d61ebd9a06bf240d594129184dc0","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114291888369438720","occurred_at":"2026-10-25T09:44:02Z","signature_id":"1187688d249cfcb086795f44465ce4133550dfe21fbda93d7619f7c1cd0c8e0e","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","signature_id":"0df12b245e7e51211310f93e12a460611030c5368b7e9d6d2e212e20fb97294f","pr_number":7733,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","failure_pattern":"not all expected service metrics appeared in Azure Monitor","search_query":"not all expected service metrics appeared in Azure Monitor","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.6342494714587738,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[1,2,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-22T11:13:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114665780556599296","occurred_at":"2026-10-26T10:29:42Z","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","pr_number":7665,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113227265989939200","occurred_at":"2026-10-22T11:13:36Z","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","pr_number":7665,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":3,"after_last_push_count":17,"trend_counts":[3,2,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-23T11:17:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n    resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n      \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n      \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n            correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n            client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n    resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n      \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n      \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n            correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n            client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1661e2326120\u003e: \n    resource group \"external-auth-cluster-km6bz4ns8fzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-km6bz4ns8fzk\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n      \"name\": \"cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:27:28.417107798Z\",\n      \"endTime\": \"2026-10-22T14:51:29.402583281Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n            correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b\n            client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1661e2326120\u003e: \n    resource group \"external-auth-cluster-km6bz4ns8fzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-km6bz4ns8fzk\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n      \"name\": \"cc17306c-6d65-4b24-bd6f-a1392d064e72\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:27:28.417107798Z\",\n      \"endTime\": \"2026-10-22T14:51:29.402583281Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72\n            correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b\n            client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x27b60e3061f8\u003e: \n    resource group \"pullsecret-test-2qfk9tbgwjpl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2qfk9tbgwjpl\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\",\n      \"name\": \"0e34d898-c038-4aae-864f-7bc35e33d52f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:22:39.714077689Z\",\n      \"endTime\": \"2026-10-22T14:46:39.512324074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n            correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d\n            client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x27b60e3061f8\u003e: \n    resource group \"pullsecret-test-2qfk9tbgwjpl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2qfk9tbgwjpl\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\",\n      \"name\": \"0e34d898-c038-4aae-864f-7bc35e33d52f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:22:39.714077689Z\",\n      \"endTime\": \"2026-10-22T14:46:39.512324074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f\n            correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d\n            client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"4e8926f9374f6904cc9826c0dee209a2e2cebf4787bc66fa48d1d8df4d4e2067","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"9470a39f6b3757be7a80681fe7b8ac52c6881c846eff19ecc805839f186e8123","pr_number":7686,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"bd7bb04cbc3d1b80cca9e3082743d7f6fb56d7eba3aaf104da4eae35eb46e10c","pr_number":7686,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x19a135b620a8\u003e: \n    resource group \"feature-aggregation-c29rndpv7j5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c29rndpv7j5c\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\",\n      \"name\": \"541f32be-fa69-4e38-80a2-84651b9746cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:58:24.120121666Z\",\n      \"endTime\": \"2026-10-28T05:22:31.687389775Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n            correlation ID: efcef01e-78c0-4c18-a8a7-6bed6c2a5d41\n            client request ID: 23dd3444-0c3c-438b-a21b-7fd27b67a689\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x19a135b620a8\u003e: \n    resource group \"feature-aggregation-c29rndpv7j5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c29rndpv7j5c\" to finish deleting: GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\",\n      \"name\": \"541f32be-fa69-4e38-80a2-84651b9746cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T04:58:24.120121666Z\",\n      \"endTime\": \"2026-10-28T05:22:31.687389775Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6816256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/541f32be-fa69-4e38-80a2-84651b9746cc\n            correlation ID: efcef01e-78c0-4c18-a8a7-6bed6c2a5d41\n            client request ID: 23dd3444-0c3c-438b-a21b-7fd27b67a689\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x381116f3a0c0\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n      \"name\": \"894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:28:30.111139221Z\",\n      \"endTime\": \"2026-10-25T11:52:32.090609688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n            correlation ID: d229c55a-e375-4f40-bb2a-f3ad97610736\n            client request ID: aa4bc9f4-d5d8-494d-94b7-252189caf6ce\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x381116f3a0c0\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-l9k6ftz85vvb\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n      \"name\": \"894117ff-70c7-42e1-b1b8-b8df80eb0d8c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:28:30.111139221Z\",\n      \"endTime\": \"2026-10-25T11:52:32.090609688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/894117ff-70c7-42e1-b1b8-b8df80eb0d8c\n            correlation ID: d229c55a-e375-4f40-bb2a-f3ad97610736\n            client request ID: aa4bc9f4-d5d8-494d-94b7-252189caf6ce\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x22bced7141e0\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n      \"name\": \"954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:21:43.829823922Z\",\n      \"endTime\": \"2026-10-25T11:45:52.860717396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n            correlation ID: dc5c263c-66f3-4b74-a013-b67fbdf80f9a\n            client request ID: 3eb7aaab-ba2a-414a-b973-c3e5012b0c78\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x22bced7141e0\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xcl2892ldqng\" to finish deleting: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n      \"name\": \"954cc982-3e76-4b31-9df6-fcc83a76d91a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:21:43.829823922Z\",\n      \"endTime\": \"2026-10-25T11:45:52.860717396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/954cc982-3e76-4b31-9df6-fcc83a76d91a\n            correlation ID: dc5c263c-66f3-4b74-a013-b67fbdf80f9a\n            client request ID: 3eb7aaab-ba2a-414a-b973-c3e5012b0c78\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"e8af4dcd98d7d05ebabf3df6907eac835155aa35d5063c6f3a85545c6af65f6c","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"080e3030be978a80e1095312a2869c7ed70474ec22ba1b8a90b8a510e75ff0d5","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"aa0717a28ea1274c87a79add45a1c3bfe3bb3254406149bd155ff10e105ee7d1","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a7b7935e0fbe062a83ac94a424d97876ea2ab08c4e53e37b439fcd600a57bd3","failure_pattern":"ERROR CODE: InvalidResource; detail message ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,2,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T10:12:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1523283f0200\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dzpqr982szjp\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"name\": \"5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:12.67761363Z\",\n      \"endTime\": \"2026-10-27T12:30:55.93851152Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n            correlation ID: d35a35bb-1a40-4d4f-9155-942bc6b994d5\n            client request ID: 483339d8-63cb-4cfb-a4bc-1a9aa9a36174\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1523283f0200\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dzpqr982szjp\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"name\": \"5c606ccb-fd8e-489a-a868-6835e0345028\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:12.67761363Z\",\n      \"endTime\": \"2026-10-27T12:30:55.93851152Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c606ccb-fd8e-489a-a868-6835e0345028\n            correlation ID: d35a35bb-1a40-4d4f-9155-942bc6b994d5\n            client request ID: 483339d8-63cb-4cfb-a4bc-1a9aa9a36174\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x45c8638e340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5vfww4h6t295\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"name\": \"1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:13.19675558Z\",\n      \"endTime\": \"2026-10-27T12:30:26.100595452Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n            correlation ID: 724394e1-dd61-4ca3-aed0-37913a4e01f3\n            client request ID: f5bcc160-8ac9-4bb8-b98f-d16472eb1f7c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x45c8638e340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5vfww4h6t295\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"name\": \"1ac0cdc9-8355-4081-91f3-1ae9204e1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:13.19675558Z\",\n      \"endTime\": \"2026-10-27T12:30:26.100595452Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ac0cdc9-8355-4081-91f3-1ae9204e1899\n            correlation ID: 724394e1-dd61-4ca3-aed0-37913a4e01f3\n            client request ID: f5bcc160-8ac9-4bb8-b98f-d16472eb1f7c\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x268662498700\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zczv78nf9ddg\" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n      \"name\": \"dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T11:20:30.318762088Z\",\n      \"endTime\": \"2026-10-22T11:33:05.673152156Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n            correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4\n            client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x268662498700\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zczv78nf9ddg\" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n      \"name\": \"dfac02ec-d27d-4ef0-b523-7cb0c0010faa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T11:20:30.318762088Z\",\n      \"endTime\": \"2026-10-22T11:33:05.673152156Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa\n            correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4\n            client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","signature_id":"55903a7e057966beaff233554a1a3ab22e6bb5909a2a13f86577c189bd43494f","pr_number":7740,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","signature_id":"82b8f0fbf27dd3db5294e829df65daa69cb7944c0cc4bdd9e9e49cf2f699cb72","pr_number":7740,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113211866976096256","occurred_at":"2026-10-22T10:12:25Z","signature_id":"2efde096b96e33606cefab787452a20bfbeac53b5b67e5735832387f8619958e","pr_number":7578,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e46fad7d5fafea60a6252610a97029d6a9421cce4f39dc628483a7323d419f95","failure_pattern":"ERROR CODE: InvalidResource; detail message DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-hd6rvlcsz446/cluster-candidate-4-23-z5pr5h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd0b2342e360\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-z5pr5h: failed waiting for cluster=\"cluster-candidate-4-23-z5pr5h\" in resourcegroup=\"rg-candidate-4-23-hd6rvlcsz446\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"name\": \"7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:16:28.588568581Z\",\n      \"endTime\": \"2026-10-27T12:27:01.432725234Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n            correlation ID: 3d3fc710-9f38-4fa0-b097-667c1de10d30\n            client request ID: 7b1eba2b-65de-404e-b3ec-50e9f1177df4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-hd6rvlcsz446/cluster-candidate-4-23-z5pr5h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd0b2342e360\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-z5pr5h: failed waiting for cluster=\"cluster-candidate-4-23-z5pr5h\" in resourcegroup=\"rg-candidate-4-23-hd6rvlcsz446\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"name\": \"7afde425-4c5d-47c7-a424-08163e41623a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:16:28.588568581Z\",\n      \"endTime\": \"2026-10-27T12:27:01.432725234Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7afde425-4c5d-47c7-a424-08163e41623a\n            correlation ID: 3d3fc710-9f38-4fa0-b097-667c1de10d30\n            client request ID: 7b1eba2b-65de-404e-b3ec-50e9f1177df4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-gcsfgvdh5qhb/cluster-candidate-4-22-2w4gz9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30d8f6930800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-2w4gz9: failed waiting for cluster=\"cluster-candidate-4-22-2w4gz9\" in resourcegroup=\"rg-candidate-4-22-gcsfgvdh5qhb\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"name\": \"74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:17:08.196779757Z\",\n      \"endTime\": \"2026-10-27T12:28:21.232176744Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n            correlation ID: d480a26c-43a2-49b4-8de4-02fc3c46d885\n            client request ID: 8e7d5833-9216-45e5-8c64-c9b98a48c0a2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-22-gcsfgvdh5qhb/cluster-candidate-4-22-2w4gz9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30d8f6930800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-2w4gz9: failed waiting for cluster=\"cluster-candidate-4-22-2w4gz9\" in resourcegroup=\"rg-candidate-4-22-gcsfgvdh5qhb\" to finish creating: GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"name\": \"74f9eae7-150f-4f86-81e4-f4ada23068e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:17:08.196779757Z\",\n      \"endTime\": \"2026-10-27T12:28:21.232176744Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9327744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74f9eae7-150f-4f86-81e4-f4ada23068e6\n            correlation ID: d480a26c-43a2-49b4-8de4-02fc3c46d885\n            client request ID: 8e7d5833-9216-45e5-8c64-c9b98a48c0a2\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-jhb5b2\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32758ffae460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-jhb5b2: failed waiting for cluster=\"cluster-zstream-4-22-jhb5b2\" in resourcegroup=\"rg-zstream-upgrade-4-22-lhz7fdj9k86z\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"name\": \"fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:17.939641835Z\",\n      \"endTime\": \"2026-10-27T12:30:15.55604778Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n            correlation ID: 046c9d63-6366-4a78-bd4e-177dad3c7de6\n            client request ID: 40011f58-fbc8-44b5-a868-12a5ed482aad\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-jhb5b2\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32758ffae460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-jhb5b2: failed waiting for cluster=\"cluster-zstream-4-22-jhb5b2\" in resourcegroup=\"rg-zstream-upgrade-4-22-lhz7fdj9k86z\" to finish creating: GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"name\": \"fd748747-ebe4-4429-9e81-de4bcbbdaddc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T12:19:17.939641835Z\",\n      \"endTime\": \"2026-10-27T12:30:15.55604778Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7472896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fd748747-ebe4-4429-9e81-de4bcbbdaddc\n            correlation ID: 046c9d63-6366-4a78-bd4e-177dad3c7de6\n            client request ID: 40011f58-fbc8-44b5-a868-12a5ed482aad\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","signature_id":"0b71c966ba6609560492817e57b68323576e947b7f4d9a9c9eeb5fc7aa08c840","pr_number":7770,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040147719327744","occurred_at":"2026-10-27T11:17:21Z","signature_id":"b76a2eb0d9fc40e5751f44dcc0f4dac27426fa6827332cd4c74ac4ab6381cab3","pr_number":7770,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115040060427472896","occurred_at":"2026-10-27T11:16:57Z","signature_id":"66506202a3c311949180fc813c9fe22cf549fe6653665a5b8f7d13e545e85d6a","pr_number":7740,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-23T11:17:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["src/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax    0x0\nrbx    0x7fc7134fd640\nrcx    0x7fc75a9bf94c\nrdx    0x6\nrdi    0xa70a\nrsi    0xa711\nrbp    0xa711\nrsp    0x7fc7134fc790\nr8     0x7fc7134fc860\nr9     0x7fc75aae54e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fc7134fcb60\nr14    0xccc45e750e0\nr15    0xffffffffffffffff\nrip    0x7fc75a9bf94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nsrc/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax    0x0\nrbx    0x7fc7134fd640\nrcx    0x7fc75a9bf94c\nrdx    0x6\nrdi    0xa70a\nrsi    0xa711\nrbp    0xa711\nrsp    0x7fc7134fc790\nr8     0x7fc7134fc860\nr9     0x7fc75aae54e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fc7134fcb60\nr14    0xccc45e750e0\nr15    0xffffffffffffffff\nrip    0x7fc75a9bf94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","7f52 bbf59cb1  ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640  00007ffd 4f7fb140  @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae  00007ffd 4f7fafaf  ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640  00007f52 bbed47f5  @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80  00000000 00000120  .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00  b406abf0 07e46d00  ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740  00000000 00000006  @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00  00007ffd 4f7fb200  ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0  00007f52 bbe89646  .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90  00007f52 bbe737f3  .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020  b406abf0 07e46d00   ........m......\n00007ffd4f7faf10: 00007f52 6dffc000  00007f52 bbf48ac4  ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax    0x0\nrbx    0x7f52bbe48740\nrcx    0x7f52bbed694c\nrdx    0x6\nrdi    0xa70b\nrsi    0xa70b\nrbp    0xa70b\nrsp    0x7ffd4f7fae30\nr8     0x7ffd4f7faf00\nr9     0x7f52bbffc4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7ffd4f7fb200\nr14    0x1a1d095421e0\nr15    0x10\nrip    0x7f52bbed694c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n7f52 bbf59cb1  ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640  00007ffd 4f7fb140  @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae  00007ffd 4f7fafaf  ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640  00007f52 bbed47f5  @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80  00000000 00000120  .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00  b406abf0 07e46d00  ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740  00000000 00000006  @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00  00007ffd 4f7fb200  ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0  00007f52 bbe89646  .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90  00007f52 bbe737f3  .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020  b406abf0 07e46d00   ........m......\n00007ffd4f7faf10: 00007f52 6dffc000  00007f52 bbf48ac4  ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000  00000000 0000000d  ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax    0x0\nrbx    0x7f52bbe48740\nrcx    0x7f52bbed694c\nrdx    0x6\nrdi    0xa70b\nrsi    0xa70b\nrbp    0xa70b\nrsp    0x7ffd4f7fae30\nr8     0x7ffd4f7faf00\nr9     0x7f52bbffc4e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7ffd4f7fb200\nr14    0x1a1d095421e0\nr15    0x10\nrip    0x7f52bbed694c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax    0x0\nrbx    0x7fd81f5da640\nrcx    0x7fd866a9c94c\nrdx    0x6\nrdi    0xa70c\nrsi    0xa715\nrbp    0xa715\nrsp    0x7fd81f5d9790\nr8     0x7fd81f5d9860\nr9     0x7fd866bc24e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fd81f5d9b60\nr14    0x1077594510e0\nr15    0xffffffffffffffff\nrip    0x7fd866a9c94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax    0x0\nrbx    0x7fd81f5da640\nrcx    0x7fd866a9c94c\nrdx    0x6\nrdi    0xa70c\nrsi    0xa715\nrbp    0xa715\nrsp    0x7fd81f5d9790\nr8     0x7fd81f5d9860\nr9     0x7fd866bc24e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fd81f5d9b60\nr14    0x1077594510e0\nr15    0xffffffffffffffff\nrip    0x7fd866a9c94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"041b58f88c07e6b6df5a78fe1b06fddf2a98976eca66b8f788c458fffb00b1ac","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"8d01f3d5bb1dffe043107fd710c91d72482c52b3696c1ab17cb3c16a039f0369","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"df0ed8cb7549a63439671f3dc385d910a4106ef5cb0378e7a7e6d06be9fbae28","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"daac4af340e3c5c5f7b3b73acded2f5292a6260c6a7fd84241614003c956947f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2b12a0180120\u003e: \n    resource group \"rg-candidate-4-22-9q8kddc4rfc6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hscqkp\" in resourcegroup=\"rg-candidate-4-22-9q8kddc4rfc6\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n      \"name\": \"46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:51:06.474152018Z\",\n      \"endTime\": \"2026-10-28T18:15:11.708953381Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n            correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d\n            client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2b12a0180120\u003e: \n    resource group \"rg-candidate-4-22-9q8kddc4rfc6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hscqkp\" in resourcegroup=\"rg-candidate-4-22-9q8kddc4rfc6\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n      \"name\": \"46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:51:06.474152018Z\",\n      \"endTime\": \"2026-10-28T18:15:11.708953381Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1\n            correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d\n            client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2a43d7c880a8\u003e: \n    resource group \"private-kas-2t2bq848xclk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-2t2bq848xclk\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\",\n      \"name\": \"827c464e-64e1-4e42-980f-59af842b0bed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:45:46.805727964Z\",\n      \"endTime\": \"2026-10-28T18:09:51.562148289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n            correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399\n            client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x2a43d7c880a8\u003e: \n    resource group \"private-kas-2t2bq848xclk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-2t2bq848xclk\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\",\n      \"name\": \"827c464e-64e1-4e42-980f-59af842b0bed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:45:46.805727964Z\",\n      \"endTime\": \"2026-10-28T18:09:51.562148289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed\n            correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399\n            client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x17fd8fc22138\u003e: \n    resource group \"kusto-logs-2tq8m2fllmvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2tq8m2fllmvs\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\",\n      \"name\": \"e027677d-2084-4178-b70c-b4e22dfc37b1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:46:42.22634078Z\",\n      \"endTime\": \"2026-10-28T18:10:50.94970388Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n            correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756\n            client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:\n    \u003c*errors.joinError | 0x17fd8fc22138\u003e: \n    resource group \"kusto-logs-2tq8m2fllmvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2tq8m2fllmvs\" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\",\n      \"name\": \"e027677d-2084-4178-b70c-b4e22dfc37b1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:46:42.22634078Z\",\n      \"endTime\": \"2026-10-28T18:10:50.94970388Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1\n            correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756\n            client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","signature_id":"0de28270d5e084b0197def0b8c3e6a8fb86eded5a4eb6ea2c68a0150e9491a22","pr_number":7777,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","signature_id":"c0d8d5a449f24fff80c60b0a909a50844791a9e6857245b7bc3a62487b829544","pr_number":7777,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7777/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115477832887963648","occurred_at":"2026-10-28T16:16:33Z","signature_id":"f42d951d643535f04588e61d133a18ff7cf5f9154020232391b99e2b14384551","pr_number":7777,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' sec... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T11:17:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a5d7d0bada0\u003e: \n    failed to create NodePool np-csnz6d: failed waiting for nodepool=\"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\" in resourcegroup=\"rg-candidate-5-0-x5kw7d69w7fz\" to finish creating: GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"name\": \"27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T22:49:05.815309572Z\",\n      \"endTime\": \"2026-10-26T23:08:05.399599048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Compute/virtualMachines/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Network/networkInterfaces/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n            correlation ID: aca41f08-bc3a-4eb0-aeee-dbec04407cb6\n            client request ID: 503f9ad7-c50a-4e8b-9147-306bbfbcb5a4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a5d7d0bada0\u003e: \n    failed to create NodePool np-csnz6d: failed waiting for nodepool=\"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\" in resourcegroup=\"rg-candidate-5-0-x5kw7d69w7fz\" to finish creating: GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"name\": \"27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T22:49:05.815309572Z\",\n      \"endTime\": \"2026-10-26T23:08:05.399599048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Compute/virtualMachines/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Network/networkInterfaces/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n            correlation ID: aca41f08-bc3a-4eb0-aeee-dbec04407cb6\n            client request ID: 503f9ad7-c50a-4e8b-9147-306bbfbcb5a4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n      \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n      \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n            correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n            client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n      \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n      \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n            correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n            client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114834707903418368","occurred_at":"2026-10-26T21:41:00Z","signature_id":"f156ee19e1bea684566a9ff95c7087780488a5460193b9635d1b6c6e9093f7f5","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"d010fa9f43f9777168167e02e99bc932a1d4357cede95310e395869051417e5a","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' se... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T11:32:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n    failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n      \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n      \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n            correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n            client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n    failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n      \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n      \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n            correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n            client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16e3bf104600\u003e: \n    failed to create NodePool np-jbvlzr: failed waiting for nodepool=\"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\" in resourcegroup=\"rg-candidate-5-1-jr4zj7tw52p6\" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n      \"name\": \"905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:36:41.547739466Z\",\n      \"endTime\": \"2026-10-22T03:55:45.436892812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n            correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b\n            client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16e3bf104600\u003e: \n    failed to create NodePool np-jbvlzr: failed waiting for nodepool=\"np-jbvlzr\" for cluster \"cluster-candidate-5-1-jbvlzr\" in resourcegroup=\"rg-candidate-5-1-jr4zj7tw52p6\" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n      \"name\": \"905e6e4d-deed-4c98-88d4-7e7d915bea9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:36:41.547739466Z\",\n      \"endTime\": \"2026-10-22T03:55:45.436892812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c\n            correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b\n            client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113594282169339904","occurred_at":"2026-10-23T11:32:00Z","signature_id":"c11a2050e197d4a44a7c09a12f94ca83ff56f060fe76b2742e4b18d0997beadb","pr_number":6718,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113092279206940672","occurred_at":"2026-10-22T02:17:11Z","signature_id":"91c10e9398dc1a118cf0ddb440a975ed51728767c12644ec941551e5b3bd3fe3","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T07:04:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x315531e198c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6tc6qf9c7brh\" to finish creating: GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n      \"name\": \"4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T11:51:14.858832489Z\",\n      \"endTime\": \"2026-10-28T12:10:21.145603579Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u9f3h2h7j3o5r2t-np-1-frcss-9d6tp: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-6tc6qf9c7brh--managed/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6tc6qf9c7brh--managed/providers/Microsoft.Network/networkInterfaces/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n            correlation ID: 80c82865-2a09-4afd-85b7-df3733e6ccd3\n            client request ID: 14042924-6e78-44be-a159-f0ae94071a00\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x315531e198c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6tc6qf9c7brh\" to finish creating: GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n      \"name\": \"4ebc64ca-e27a-4569-a3bf-84d10df15019\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T11:51:14.858832489Z\",\n      \"endTime\": \"2026-10-28T12:10:21.145603579Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u9f3h2h7j3o5r2t-np-1-frcss-9d6tp: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-6tc6qf9c7brh--managed/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6tc6qf9c7brh--managed/providers/Microsoft.Network/networkInterfaces/u9f3h2h7j3o5r2t-np-1-frcss-9d6tp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6407296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4ebc64ca-e27a-4569-a3bf-84d10df15019\n            correlation ID: 80c82865-2a09-4afd-85b7-df3733e6ccd3\n            client request ID: 14042924-6e78-44be-a159-f0ae94071a00\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xed2a17d5560\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kr8h8mgbckw7\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\",\n      \"name\": \"a192dd85-8881-406d-a906-718d3d901c7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T08:02:04.742645812Z\",\n      \"endTime\": \"2026-10-22T08:21:04.297195938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n            correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75\n            client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xed2a17d5560\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-kr8h8mgbckw7\" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\",\n      \"name\": \"a192dd85-8881-406d-a906-718d3d901c7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T08:02:04.742645812Z\",\n      \"endTime\": \"2026-10-22T08:21:04.297195938Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e\n            correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75\n            client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115394496836407296","occurred_at":"2026-10-28T10:45:24Z","signature_id":"7e41b58372eff808bb4c5baa7c863d948b46c4a604a47df1656a755e8ad17000","pr_number":7783,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","signature_id":"5aa430a71f1e1be5ff2471115ee092e684da6d03063af5eec54d8c3b6e8e964a","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T00:01:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27856e5eac20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xpvzdt8vb9wc\" to finish creating: GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"name\": \"1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:39:39.672372928Z\",\n      \"endTime\": \"2026-10-27T21:58:43.258733633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7h4b0s5o2n2n4p-np-1-29gbc-4c99s: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xpvzdt8vb9wc--managed/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xpvzdt8vb9wc--managed/providers/Microsoft.Network/networkInterfaces/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n            correlation ID: c4cb1e97-d5ea-4bde-b8e9-0601e9a9af7f\n            client request ID: b322bb9e-3c1b-45b3-9299-27b3632803d2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27856e5eac20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xpvzdt8vb9wc\" to finish creating: GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"name\": \"1be16755-3c78-4402-8a77-c15c787ee7a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:39:39.672372928Z\",\n      \"endTime\": \"2026-10-27T21:58:43.258733633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7h4b0s5o2n2n4p-np-1-29gbc-4c99s: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xpvzdt8vb9wc--managed/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xpvzdt8vb9wc--managed/providers/Microsoft.Network/networkInterfaces/v7h4b0s5o2n2n4p-np-1-29gbc-4c99s-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4268416.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1be16755-3c78-4402-8a77-c15c787ee7a6\n            correlation ID: c4cb1e97-d5ea-4bde-b8e9-0601e9a9af7f\n            client request ID: b322bb9e-3c1b-45b3-9299-27b3632803d2\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n      \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n      \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n            correlation ID: 980d0943-0968-489f-acea-10a235919d44\n            client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n      \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n      \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n            correlation ID: 980d0943-0968-489f-acea-10a235919d44\n            client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115181315694268416","occurred_at":"2026-10-27T20:38:18Z","signature_id":"680a2853e0b95ed98a216673aae885a5a987ff0bbda27f4f85819f506da3a15f","pr_number":7786,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"0d9828b0bb1f27d9cc3639f58f494971e86d707f6ec7be8d51e1bfc562947fba","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"75e25b16c89c5bd443d1560c5ffb0c0789ec4d49846a07583c802aab771d1e1c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AssociationFailedAtResourceProvider; detail message Association provisioning failed at resource provider; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-28T22:46:09.268Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-28T22:46:09.760Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1\ntime=2026-10-28T22:46:10.194Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 deployment=351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9857152-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2\ntime=2026-10-28T22:47:10.427Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Resources/deployments/351403f99eccc84e000639428bd1b6b39bdb6ecf570569167f5f1b17cdb7aee2/operationStatuses/08584083793155131248\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Resources/deployments/profile-ufpn5r2pwmb4i\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: 824ad833-5117-4ca4-a2e6-1002e46997d8\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9857152-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9857152-mgmt-1/resourceAssociations/nsp-j9857152-mgmt-1-ra-67927217-e211-550f-abbf-b3e73881a01a\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AssociationFailedAtResourceProvider\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Association provisioning failed at resource provider\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-28T13:56:43.120Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-28T13:56:43.546Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-10-28T13:56:43.936Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7323392-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244\ntime=2026-10-28T13:58:14.248Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Resources/deployments/27369930d49e6b87bdd6e0a76bb435d03ceef9149d6c362d06dfe1d47f9f5244/operationStatuses/08584084110817295270\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Resources/deployments/profile-sxizw5tprl6fg\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7323392-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7323392-mgmt-2/resourceAssociations/nsp-j7323392-mgmt-2-ra-4bf3dc7e-4183-5ff8-b0f7-92d18b1331a1\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AssociationFailedAtResourceProvider\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Association provisioning failed at resource provider\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115560273719857152","occurred_at":"2026-10-28T21:44:09Z","signature_id":"6c11c44592820325fa1934724e67fb473057c5c4359853d87ec5141e3481ef08","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115431844907323392","occurred_at":"2026-10-28T13:13:49Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":7783,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-13T15:32:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":2}],"full_error_samples":["alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8739840/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j8739840' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2704640/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j2704640' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7793/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115451033688739840","occurred_at":"2026-10-28T14:30:04Z","signature_id":"d449b94908925109550a95da2d5df23c82ed852f6115ba04b76530f4e281f137","pr_number":7793,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115257563342704640","occurred_at":"2026-10-28T01:41:17Z","signature_id":"02713fa7ad9d7fa3c69367cb802a9482f205704d583c9d5a19a7d63dd9f94544","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca3f18ba576fbe4160cedc95bb389561f1f95766f5e0b5c110315741f9671852","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-24T12:30:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2tqhb4\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2f8a3019d120\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2f8a3019d0e0\u003e{\n            observation: \u003c*errors.errorString | 0x2f8a325804f0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2f8a3019cf00\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2tqhb4\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2f8a3019d120\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x2f8a3019d0e0\u003e{\n            observation: \u003c*errors.errorString | 0x2f8a325804f0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2f8a3019cf00\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2ll92l\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xd8e10b4e180\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0xd8e11f02690\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2ll92l\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xd8e10b4e180\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0xd8e11f02690\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","signature_id":"2560966459f92d4f35ec9c0d4f05ec298c951267691f5f9a53995585f0aca9dd","pr_number":7734,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","signature_id":"891240bac405e3baa2932f2cd07a3c04a37450b93782bafba1074f2d0e0d8a3d","pr_number":7734,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cd80d6b45c8eb9b1b33eb30b789a71289e43bb10694295e955e5ee50ed8fa41","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16b125c17c20\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x16b125c17c00\u003e{\n            observation: \u003c*errors.errorString | 0x16b125e61c20\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x16b125c17b80\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x16b1258ee6e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x16b125c17b20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x16b127c385d0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16b125c17c20\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9nrkl9lvccng/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x16b125c17c00\u003e{\n            observation: \u003c*errors.errorString | 0x16b125e61c20\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x16b125c17b80\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x16b1258ee6e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x16b125c17b20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j7684992-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnf2nfhj53l39qtduvs91q7n2s4eeqa-n3o6u1n6i9x0d6i\\\"; let start = datetime(2026-10-28T00:18:02.1234737Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x16b127c385d0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2148d2b87580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2148d2b87560\u003e{\n            observation: \u003c*errors.errorString | 0x2148d2b63740\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2148d2b87540\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2148d2b84b90\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2148d2b87500\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2148d332b860\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2148d2b87580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2148d2b87560\u003e{\n            observation: \u003c*errors.errorString | 0x2148d2b63740\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2148d2b87540\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2148d2b84b90\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2148d2b87500\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2148d332b860\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216183337684992","occurred_at":"2026-10-27T22:56:51Z","signature_id":"4b6ad7abc4a4e91efe1a1b87dfaec822083c16eb358fb11b4a36c244a76d6e3c","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114823017199570944","occurred_at":"2026-10-26T20:54:33Z","signature_id":"e7ac52e3e916ffb1ddc5bfb303098335d5fc083fc3a4ba8e96d544894507de5c","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6249e15d0a6c7cf0142092c6b9eca0eb8f84826824bb608613ac26e30d3b14b0","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x67403d76ae0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x67403d76ac0\u003e{\n            observation: \u003c*errors.errorString | 0x674032428b0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x67403d76aa0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x674036d06e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x67403d76a80\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x67404ab4ba0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x67403d76ae0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w96mrp6lpnz7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x67403d76ac0\u003e{\n            observation: \u003c*errors.errorString | 0x674032428b0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x67403d76aa0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x674036d06e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x67403d76a80\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j5001984-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnvsselnr4pgg1e0n2skr2pggevm4b6-w8s3q8g2g8v3s0d\\\"; let start = datetime(2026-10-28T19:24:48.1549118Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x67404ab4ba0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17b1db069120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0x17b1da3a61e0\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17b1db069120\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mc9l9bgvmrc7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0x17b1da3a61e0\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509651545001984","occurred_at":"2026-10-28T18:22:59Z","signature_id":"d604f64973262ef8b5c3dcda3c973088263ebb3e1c8041dd41bfb3a67446230c","pr_number":7814,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115035820850155520","occurred_at":"2026-10-27T11:00:09Z","signature_id":"16ae8d5d9b50963b314bb9a1ebf572dcd3f3137a44de5de8da13239140ef0aec","pr_number":7773,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c8baf4b2e1a6c519c2e6a72beff1cb671e4fa39d2c9d8669c830590e4eb74e17","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node \u003cnode\u003e (reason=ProvisioningFailed, count=8, age=7m53s)","search_query":"[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-cptgtj\"\nUnexpected error:\n    \u003c*errors.joinError | 0x199ff3994258\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\n    [pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\n    [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-s4t9m/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x199ff3d7a160\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n                err: \u003c*fmt.wrapError | 0x199ff3d7a020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x199ff3d7a680\u003e{\n                        observation: \u003c*errors.errorString | 0x199ff36faf30\u003e{\n                            s: \"pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x199ff3d7a660\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-s4t9m/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-cptgtj\"\nUnexpected error:\n    \u003c*errors.joinError | 0x199ff3994258\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\n    [pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\n    [events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-s4t9m/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x199ff3d7a160\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n                err: \u003c*fmt.wrapError | 0x199ff3d7a020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.006s: pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-s4t9m/e2e-csi-pvc phase=Bound volumeName=pvc-742751d0-4b4e-4c69-b7d6-859886d48626 storageClass=managed-csi\\n[pod] e2e-csi-disk-s4t9m/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-s4t9m/e2e-csi-pod to l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=Scheduled, count=0, age=5m24s)\\n[events] e2e-csi-disk-s4t9m Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-742751d0-4b4e-4c69-b7d6-859886d48626\\\" : rpc error: code = Internal desc = Attach volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 to instance l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 failed with throttled due to too many requests (reason=FailedAttachVolume, count=9, age=2m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=19, age=5m33s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-s4t9m/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node l1m3f0i9q3m4q6d-np-cptgtj-w9vkt-7rxb5 (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-s4t9m PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-742751d0-4b4e-4c69-b7d6-859886d48626 (reason=ProvisioningSucceeded, count=1, age=5m24s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x199ff3d7a680\u003e{\n                        observation: \u003c*errors.errorString | 0x199ff36faf30\u003e{\n                            s: \"pod e2e-csi-disk-s4t9m/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x199ff3d7a660\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-s4t9m/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-94pk4p\"\nUnexpected error:\n    \u003c*errors.joinError | 0x380579046d8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\n    [pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-fj4vw/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x38056a72920\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                err: \u003c*fmt.wrapError | 0x38056fdc300\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x38056fb6b80\u003e{\n                        observation: \u003c*errors.errorString | 0x380577528d0\u003e{\n                            s: \"pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x38056fb6b20\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-fj4vw/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-94pk4p\"\nUnexpected error:\n    \u003c*errors.joinError | 0x380579046d8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\n    [pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\n    [events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-fj4vw/e2e-csi-pvc\" (reason=Provisioning, count=9, age=5m45s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\n    [events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x38056a72920\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                err: \u003c*fmt.wrapError | 0x38056fdc300\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-fj4vw/e2e-csi-pvc phase=Bound volumeName=pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be storageClass=managed-csi\\n[pod] e2e-csi-disk-fj4vw/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-fj4vw/e2e-csi-pod to s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=Scheduled, count=0, age=5m21s)\\n[events] e2e-csi-disk-fj4vw Pod/e2e-csi-pod: AttachVolume.Attach failed for volume \\\"pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be\\\" : rpc error: code = Internal desc = could not get disk lun for volume /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed/providers/Microsoft.Compute/disks/pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be: getNodeVMSet: failed to check the node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks management type: getter function of nonVmssUniformNodesCache: failed to list vms in the resource group rg-candidate-4-23-mdwfnk5tx96c-candidate-94pk4p--managed: throttled due to too many requests (reason=FailedAttachVolume, count=9, age=1m58s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m1s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=1, age=10m0s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-fj4vw/e2e-csi-pvc\\\" (reason=Provisioning, count=9, age=5m45s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node s8m2i8d5f5w8n1d-np-94pk4p-q9sxk-psnks (reason=ProvisioningFailed, count=8, age=7m53s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=17, age=5m36s)\\n[events] e2e-csi-disk-fj4vw PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-9120227e-fe48-4e13-8ec5-8e95f21af7be (reason=ProvisioningSucceeded, count=1, age=5m22s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x38056fb6b80\u003e{\n                        observation: \u003c*errors.errorString | 0x380577528d0\u003e{\n                            s: \"pod e2e-csi-disk-fj4vw/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x38056fb6b20\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-fj4vw/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115462331566657536","occurred_at":"2026-10-28T15:14:57Z","signature_id":"4037be027d723586834ec2417ff728f3a460e286ca670e3bf3147bcf80036590","pr_number":7802,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115090302321561600","occurred_at":"2026-10-27T14:36:39Z","signature_id":"6ad8d70ef0c34180dc85202729c1b6c4224cb11128f69d82ce5269b20ff050bd","pr_number":7705,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":20,"prior_last_seen_at":"2026-10-23T01:45:26Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-23T02:40:08Z\n  Ended:       2026-10-23T02:49:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3977984-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T21:15:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3800704-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T21:24:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","signature_id":"21ef629d9815e1be0aa9ab1b5d0b1208b139615a20d5ce194bf36c9d123c0ea8","pr_number":7703,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"73605e60345d9768cbfbea26978720be4c3942667f138969f6f3eb62f002e92e","failure_pattern":"alert [svc] userJourneyBackupMetricsAbsent fired","search_query":"alert [svc] userJourneyBackupMetricsAbsent fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyBackupMetricsAbsent does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyBackupMetricsAbsent does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T00:19:29Z\n  Ended:       2026-10-28T00:26:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j8270720-mgmt-2\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j8270720-mgmt-2 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-28T00:20:31Z\n  Ended:       2026-10-28T00:26:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j8270720-mgmt-1\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j8270720-mgmt-1 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-27T15:59:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyBackupMetricsAbsent\", cluster=\"ci01-j5601792-mgmt-2\", component=\"slo\", severity=\"3\", slo=\"backup-metrics-absent\"\n  Description: Management cluster ci01-j5601792-mgmt-2 is in the underlay-cluster inventory but has had no successful Velero server metrics scrape for 15m. Check Velero, its PodMonitor, and the management-cluster metrics pipeline. A healthy empty cluster with up=1 is not a failure merely because it has no backup attempts."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","signature_id":"c0be5c770591fc9adef802c76d1c98311b28c10ee1c66cb27508a32677e60094","pr_number":5244,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115093743005601792","occurred_at":"2026-10-27T14:50:19Z","signature_id":"9f8170d470887f01d9dcbfc134faa8684787533bfd7b84d71b542971a3f5c05d","pr_number":7670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b616f5a6866cb92586bb6c324d97afd731ae9a87974504a140606af8f1c6d8c7","failure_pattern":"error code: CANCEL","search_query":"error code: CANCEL","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-th8csc\"\nUnexpected error:\n    \u003c*errors.joinError | 0x28585f5aa1e0\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\n    [pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-5zpk7/e2e-csi-pvc\" (reason=Provisioning, count=10, age=36s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28585f462380\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n                err: \u003c*fmt.wrapError | 0x28585e8a4800\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x28585e8a43c0\u003e{\n                        observation: \u003c*errors.errorString | 0x28585ee0a670\u003e{\n                            s: \"PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x28585e8a4320\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-5zpk7/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-th8csc\"\nUnexpected error:\n    \u003c*errors.joinError | 0x28585f5aa1e0\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\n    [pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-5zpk7/e2e-csi-pvc\" (reason=Provisioning, count=10, age=36s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\n    [events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28585f462380\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n                err: \u003c*fmt.wrapError | 0x28585e8a4800\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.004s: PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-5zpk7/e2e-csi-pvc phase=Bound volumeName=pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f storageClass=managed-csi\\n[pod] e2e-csi-disk-5zpk7/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-5zpk7 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-5zpk7/e2e-csi-pod to t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=Scheduled, count=0, age=12s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-5zpk7/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=36s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node t8z9t1e8w4t1q5i-np-th8csc-h6n6d-49dfg (reason=ProvisioningFailed, count=5, age=9m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m48s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=8m47s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=3m33s)\\n[events] e2e-csi-disk-5zpk7 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-ec65a1b9-74c2-42c0-961c-c9b41647402f (reason=ProvisioningSucceeded, count=1, age=13s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x28585e8a43c0\u003e{\n                        observation: \u003c*errors.errorString | 0x28585ee0a670\u003e{\n                            s: \"PVC e2e-csi-disk-5zpk7/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x28585e8a4320\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-5zpk7/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-wcdhz7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13c32a30a4f8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-2drrj/e2e-csi-pvc\" (reason=Provisioning, count=10, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13c329ac69a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                err: \u003c*fmt.wrapError | 0x13c329f00fa0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x13c329f00aa0\u003e{\n                        observation: \u003c*errors.errorString | 0x13c329cc21b0\u003e{\n                            s: \"PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x13c329f00a60\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-2drrj/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-wcdhz7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13c32a30a4f8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-2drrj/e2e-csi-pvc\" (reason=Provisioning, count=10, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\n    [events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13c329ac69a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                err: \u003c*fmt.wrapError | 0x13c329f00fa0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-2drrj/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-2drrj/e2e-csi-pod phase=Pending reason= message=\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=25, age=4m6s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-2drrj/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node q3m2n2b2b2o7h2b-np-wcdhz7-vsftl-w7sqs (reason=ProvisioningFailed, count=4, age=9m53s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=5, age=3m45s)\\n[events] e2e-csi-disk-2drrj PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL (reason=ProvisioningFailed, count=1, age=3m15s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x13c329f00aa0\u003e{\n                        observation: \u003c*errors.errorString | 0x13c329cc21b0\u003e{\n                            s: \"PVC e2e-csi-disk-2drrj/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x13c329f00a60\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-2drrj/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7802/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409868675354624","occurred_at":"2026-10-28T11:46:29Z","signature_id":"70032133cd194d044b701e3ebacbe2526af4f272899063e74ddee1c469f750e1","pr_number":7802,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025512358219776","occurred_at":"2026-10-27T10:19:11Z","signature_id":"2787f66c9d2a7a1f7a92b3a296589aebcd6aaf28beaeae1caae03bb3fb461214","pr_number":7615,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c2e7122f012fcda98e11e9f11c6eb72558008d2e2cc0835ce25a3969a080ede6","failure_pattern":"failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T13:03:13Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce (stamp=1)","occurrences":2}],"full_error_samples":["time=2026-10-27T13:38:02.530Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-10-27T13:38:03.129Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-10-27T13:38:03.129Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:38:03.342Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 namespace=multicluster-engine\ntime=2026-10-27T13:38:03.472Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 dryRun=false\ntime=2026-10-27T13:38:03.472Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:38:03.599Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-27T13:43:12.521Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-27T13:43:12.521Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"","time=2026-10-22T13:58:37.009Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-10-22T13:58:37.626Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-10-22T13:58:37.626Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T13:58:37.853Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 namespace=multicluster-engine\ntime=2026-10-22T13:58:38.094Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 dryRun=false\ntime=2026-10-22T13:58:38.094Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T13:58:38.280Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1\ntime=2026-10-22T14:03:45.649Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-22T14:03:45.650Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115063427364818944","occurred_at":"2026-10-27T12:49:51Z","signature_id":"6226765797caa992638fa67211f61137351872a74ebe70cc5200677de47ce171","pr_number":6823,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113254849540788224","occurred_at":"2026-10-22T13:03:13Z","signature_id":"6226765797caa992638fa67211f61137351872a74ebe70cc5200677de47ce171","pr_number":7682,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c3bac01c69d371179a684cc643681dbf473300fa462d0ddc425d19fb5a7bff64","failure_pattern":"✗ Container group swift-vnet-aks-net failed to provision","search_query":"✗ Container group swift-vnet-aks-net failed to provision","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.42283298097251587,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T20:45:37Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-22T21:15:19.830Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-10-22T21:15:40.577Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 2s)...\\n✗ Container group swift-vnet-aks-net failed to provision\\n\\n exit status 1\"","time=2026-10-22T18:49:27.700Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-10-22T18:49:46.822Z level=ERROR msg=\"failed to remove step Azure CLI config directory\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"unlinkat /tmp/azure-cli-step-config-1336382318: directory not empty\" dir=/tmp/azure-cli-step-config-1336382318\ntime=2026-10-22T18:49:46.822Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 1s)...\\n✗ Container group swift-vnet-aks-net failed to provision\\n\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113371217632169984","occurred_at":"2026-10-22T20:45:37Z","signature_id":"7dba303763564f512f9c5cae72718a1ea571c1a5f6ff70b0d0efd7d6d4c2edaa","pr_number":7119,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7697/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113336312449732608","occurred_at":"2026-10-22T18:26:55Z","signature_id":"980c914d74e054017fdcac7ea3a9f64034c21dfbdbe2586220b7dbcbd475b756","pr_number":7697,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1f45b2fce5a9656a648589df07358b5c8f3221613247fa4e2a6dfc2fdd8f6c58","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f211b784120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fqjfphzpbt4x\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n      \"name\": \"dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:22.163428437Z\",\n      \"endTime\": \"2026-10-25T11:30:25.784218117Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 19m44s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n            correlation ID: 16b92b93-ef61-4f4d-9bbb-a63f0c7ebf4f\n            client request ID: cc5cc315-91ce-4032-81e3-c0e49c80e860\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f211b784120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fqjfphzpbt4x\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n      \"name\": \"dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:22.163428437Z\",\n      \"endTime\": \"2026-10-25T11:30:25.784218117Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 19m44s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc5b2708-f9a2-4d86-915e-eb8ba91cbd59\n            correlation ID: 16b92b93-ef61-4f4d-9bbb-a63f0c7ebf4f\n            client request ID: cc5cc315-91ce-4032-81e3-c0e49c80e860\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x299d616e8140\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bcvsvbp5d8t7\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n      \"name\": \"f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:32.174992685Z\",\n      \"endTime\": \"2026-10-25T11:30:36.037388523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m56s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n            correlation ID: 88e2d1ef-c0a9-47ed-aaa6-0693add21a52\n            client request ID: 8ab0d68c-218c-40c0-b734-d3980ba8ecab\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x299d616e8140\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-bcvsvbp5d8t7\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n      \"name\": \"f4d6594c-1ee5-4e01-824a-b335b8b1a671\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:32.174992685Z\",\n      \"endTime\": \"2026-10-25T11:30:36.037388523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m56s ago; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4d6594c-1ee5-4e01-824a-b335b8b1a671\n            correlation ID: 88e2d1ef-c0a9-47ed-aaa6-0693add21a52\n            client request ID: 8ab0d68c-218c-40c0-b734-d3980ba8ecab\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"782e2be5be125f055ff2ce43c736756398d39d6effe0fee98a9961842d3d84c5","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"fb0f5c839574ca03941c2ecf3ef4dfd30b5ba58990ab84508676591669d4a42b","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-22T18:39:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x5f7edbba210\u003e: \n    resource group \"feature-aggregation-9bffc7pfd9cf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9bffc7pfd9cf\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\",\n      \"name\": \"80845250-6487-4bd8-bc2d-fcba49147d69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T20:00:56.002177295Z\",\n      \"endTime\": \"2026-10-22T20:25:04.157804188Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n            correlation ID: b47a1ca2-3039-4616-a8a8-424022677398\n            client request ID: da4cb843-dd82-426b-ae29-8754ca365c47\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x5f7edbba210\u003e: \n    resource group \"feature-aggregation-9bffc7pfd9cf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9bffc7pfd9cf\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\",\n      \"name\": \"80845250-6487-4bd8-bc2d-fcba49147d69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T20:00:56.002177295Z\",\n      \"endTime\": \"2026-10-22T20:25:04.157804188Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69\n            correlation ID: b47a1ca2-3039-4616-a8a8-424022677398\n            client request ID: da4cb843-dd82-426b-ae29-8754ca365c47\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3e4a7eaec120\u003e: \n    resource group \"kusto-logs-bjqbcmj9fh99\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjqbcmj9fh99\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n      \"name\": \"133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T20:01:21.775547749Z\",\n      \"endTime\": \"2026-10-22T20:25:24.154021571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n            correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2\n            client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3e4a7eaec120\u003e: \n    resource group \"kusto-logs-bjqbcmj9fh99\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bjqbcmj9fh99\" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n      \"name\": \"133a0c79-b758-4022-b9d3-abe3437fdb7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T20:01:21.775547749Z\",\n      \"endTime\": \"2026-10-22T20:25:24.154021571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e\n            correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2\n            client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"8cfd4f8b1047556e1120af67bd86078b72621252b89d930d39333adaf2164a5c","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113339463437389824","occurred_at":"2026-10-22T18:39:26Z","signature_id":"bfa94f3d303830ea50f4ae4f2810d453787700edaab1bd594d0d227e0fd34552","pr_number":6928,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-23T05:21:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","pr_number":7463,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e081efea8bc058d3864749e2214faab4cad1dba916314a282090b76fc0651463","failure_pattern":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T04:13:32Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","occurrences":1}],"full_error_samples":["time=2026-10-23T04:51:07.246Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-23T04:52:17.695Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry registry.stage.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc.\\nThe image will still be available under it's original digest sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc in the target registry.\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113483940428779520","occurred_at":"2026-10-23T04:13:32Z","signature_id":"5351510afe6cf56e3a8b6c40bacbd2bff16093b6e23691ab45c119900b535d4b","pr_number":6770,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-22T12:57:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x291accb9c480\u003e: \n    resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x291acc2cd520\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-n5m774zrgzv9\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x291acc2cd500\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x291acd993a70\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x291acd993a40\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x291acd69fed0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x291acd993a40\u003e0x291acd993a40 (cyclic reference),\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x291accb9c480\u003e: \n    resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x291acc2cd520\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-n5m774zrgzv9\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x291acc2cd500\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x291acd993a70\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x291acd993a40\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-n5m774zrgzv9\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x291acd69fed0\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x291acd993a40\u003e0x291acd993a40 (cyclic reference),\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"a420d5a9e3f855f48c2651db4f74b1059d2169fa2a73c735b46816a88994b8a8","pr_number":7686,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5e13e8ebe7676d236f83bd844b8c7a828a419da28309fcb2c22c74d629e1c635","failure_pattern":"Denial should come from the image registry policy, got: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Image Registry Policy should deny pods with images from disallowed registries","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T12:01:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Image Registry Policy should deny pods with images from disallowed registries","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n    \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n    \u003cstring\u003e: not from an allowed registry\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n    \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n    \u003cstring\u003e: not from an allowed registry"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113601669802496000","occurred_at":"2026-10-23T12:01:21Z","signature_id":"f653863f476c685d13e3a7415f0b0b11a8654a0f134f21f96f43a33cb5375be8","pr_number":7611,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"11f1e4870cb02ff7036064941db2a3263ef1df1d64949078bceae777a2c26ab6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T11:52:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n    failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n      \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n      \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n            correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n            client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n    failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n      \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n      \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n            correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n            client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","signature_id":"4b66edeccfb70c1f5cfd0f1d6ba0073696d1cd6c7003d445193ae337daf93d50","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"146ba2f03b8e73d8b9aa488d4c220ca3b04255c2372536bbb59e6aa3a97d9e10","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Ple... message truncated Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T06:15:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n      \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n      \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n            correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n            client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n      \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n      \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n            correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n            client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"cbafe1b07c5b593f659af5b88993e3571384544466e1095e0be4c9ecc5336c65","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0f63bd757f8c437f8d2808c42bcc57ab70f3b2125a2a84565f38238af08e7d10","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try ag... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f06fbed83a0\u003e: \n    failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-4lmptk5vpx7s\" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\",\n      \"name\": \"f4b98636-6475-41ec-a151-007f2ef479ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:44:01.234316083Z\",\n      \"endTime\": \"2026-10-28T20:03:07.974415443Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n            correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56\n            client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f06fbed83a0\u003e: \n    failed to create NodePool swift-workers: failed waiting for nodepool=\"swift-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-4lmptk5vpx7s\" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\",\n      \"name\": \"f4b98636-6475-41ec-a151-007f2ef479ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:44:01.234316083Z\",\n      \"endTime\": \"2026-10-28T20:03:07.974415443Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea\n            correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56\n            client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7816/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115512185487626240","occurred_at":"2026-10-28T18:33:03Z","signature_id":"eed1cf90a87829b734d34d65add9b5f9a3ad3af46df74a86ca0e186f05eb617f","pr_number":7816,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"822bfeba7eb34370c8d8683ec2656c86acd96391b5948409cfb9754f6ed523ca","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' s... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34a8b7c1a9e0\u003e: \n    failed to create NodePool np-8844z2: failed waiting for nodepool=\"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\" in resourcegroup=\"rg-candidate-4-22-p8n8vcw2knj6\" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\",\n      \"name\": \"e163bbab-c728-4e64-85d2-39934825ad2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:07:17.250607245Z\",\n      \"endTime\": \"2026-10-28T17:26:25.565644744Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n            correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3\n            client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34a8b7c1a9e0\u003e: \n    failed to create NodePool np-8844z2: failed waiting for nodepool=\"np-8844z2\" for cluster \"cluster-candidate-4-22-8844z2\" in resourcegroup=\"rg-candidate-4-22-p8n8vcw2knj6\" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\",\n      \"name\": \"e163bbab-c728-4e64-85d2-39934825ad2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T17:07:17.250607245Z\",\n      \"endTime\": \"2026-10-28T17:26:25.565644744Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c\n            correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3\n            client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115472312617668608","occurred_at":"2026-10-28T15:54:37Z","signature_id":"d8a52ca95428cb84af728b6065a7e821205b568d8191e5dba7ff7cf5f9250e85","pr_number":7757,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6df737c7181b1e617209864731522070f8aa374ebcc148681c0698c5ad2ea926","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-20T17:21:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfe103ee9c0\u003e: \n    failed to create NodePool np-m4xxs2: failed waiting for nodepool=\"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\" in resourcegroup=\"rg-candidate-4-22-9hlfn44n2sw7\" to finish creating: GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"name\": \"fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T15:26:37.894672503Z\",\n      \"endTime\": \"2026-10-27T15:45:41.452550561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Compute/virtualMachines/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Network/networkInterfaces/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n            correlation ID: c84286ce-64a0-40f9-bed8-eadb6c3c019f\n            client request ID: 39555e0e-ee7f-4ff0-9056-9ab0bae5c62d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfe103ee9c0\u003e: \n    failed to create NodePool np-m4xxs2: failed waiting for nodepool=\"np-m4xxs2\" for cluster \"cluster-candidate-4-22-m4xxs2\" in resourcegroup=\"rg-candidate-4-22-9hlfn44n2sw7\" to finish creating: GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"name\": \"fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T15:26:37.894672503Z\",\n      \"endTime\": \"2026-10-27T15:45:41.452550561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Compute/virtualMachines/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-9hlfn44n2sw7-candidate-m4xxs2--managed/providers/Microsoft.Network/networkInterfaces/m5b2t7u3p0b4b5s-np-m4xxs2-7dxm2-5dghn-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4902016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fd0c6ffb-fa75-4ec9-bacd-47340ee2ccc0\n            correlation ID: c84286ce-64a0-40f9-bed8-eadb6c3c019f\n            client request ID: 39555e0e-ee7f-4ff0-9056-9ab0bae5c62d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115085319714902016","occurred_at":"2026-10-27T14:16:47Z","signature_id":"f3dc95e2199f380975340c82cae7dc3176d54456533ece2dce8ed30b45782845","pr_number":7769,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"802121af546da5e12fc3cac43502a22a348ae1f3e72897092323375f66a04dbc","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after addi... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T11:52:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n      \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n      \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n            correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n            client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n      \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n      \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n            correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n            client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","signature_id":"b2e5ac734561cd50d6d2bb64f76bb0716f87849cddae08ac51680aa3c5151ea8","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0a3bf853ee2796c58618727ae662e023afe77aeea1b8cb1ea78a9c68517783b0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional t... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4218f58d8c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8q6szjwkq6rk\" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n      \"name\": \"78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:13:11.691142567Z\",\n      \"endTime\": \"2026-10-28T21:32:17.541374372Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n            correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d\n            client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4218f58d8c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8q6szjwkq6rk\" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n      \"name\": \"78051d0a-b2ad-4647-99d4-120efa3f6a99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:13:11.691142567Z\",\n      \"endTime\": \"2026-10-28T21:32:17.541374372Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99\n            correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d\n            client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115536999510183936","occurred_at":"2026-10-28T20:11:40Z","signature_id":"ea34188856e004cdd568b90c2383e830d4cf10e6aa357acf563621e40e9840a2","pr_number":7809,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-20T15:36:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76964b22320\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-x8bmdggt6rx8\" to finish creating: GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"name\": \"589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:30:16.899807125Z\",\n      \"endTime\": \"2026-10-28T00:49:22.446308948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-6hzwl-4vtdh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-x8bmdggt6rx8--managed/ea-cluster-ea-np-1-6hzwl-4vtdh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-6hzwl-4vtdh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n            correlation ID: 10498359-e6e2-4fc8-aa1b-45727688742e\n            client request ID: 1f6fa3eb-c07d-45c3-990e-463cf99216e9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76964b22320\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-x8bmdggt6rx8\" to finish creating: GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"name\": \"589efcf6-5fd0-4d8a-b52e-89de192a8b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T00:30:16.899807125Z\",\n      \"endTime\": \"2026-10-28T00:49:22.446308948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-6hzwl-4vtdh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-6hzwl-4vtdh: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-x8bmdggt6rx8--managed/ea-cluster-ea-np-1-6hzwl-4vtdh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-x8bmdggt6rx8--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-6hzwl-4vtdh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7684992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/589efcf6-5fd0-4d8a-b52e-89de192a8b64\n            correlation ID: 10498359-e6e2-4fc8-aa1b-45727688742e\n            client request ID: 1f6fa3eb-c07d-45c3-990e-463cf99216e9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115216183337684992","occurred_at":"2026-10-27T22:56:51Z","signature_id":"9f72ae7d5870df0c4286d045d51fbd4b7ca3a022b871ba5133e06bc50029740a","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9ecd96fd76a659f151ac5df30d476a4d8045c25ee424b28225788c72e8971d2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refe... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T08:20:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n      \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n      \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n            correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n            client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n      \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n      \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n            correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n            client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","signature_id":"40c4710e56e4b7283917fa74062c14639587a07ba8e4e6a92442bf16cdee182b","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6a41a55832324284d10413d9a99da8343f3b8c1a9dc53d5ecf869bc08ac34982","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer t... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T15:09:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n      \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n      \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n            correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n            client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n      \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n      \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n            correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n            client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113649065022459904","occurred_at":"2026-10-23T15:09:40Z","signature_id":"1ddc8754dde4d08e908579fef94b5559366fe2b19d0cf98ab8ba070611484821","pr_number":7656,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5bd4868e5b799ebb3599c1c4a1a9e5adf369cd18ab421a17e5fc658ab1e93070","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T08:30:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n      \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n      \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n            correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n            client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n      \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n      \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n            correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n            client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113548717339447296","occurred_at":"2026-10-23T08:30:48Z","signature_id":"eac87251b067df59431cf544f3a76dfd222fded34dfba939a62a417723a28785","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55f249c389c8381077274234c503c49f7f7b8734eaf356caba055061beb868f8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[2,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-21T18:57:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x141a49f31000\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-czkfqz2c2dgm\" to finish creating: GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"name\": \"8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:03:30.992910534Z\",\n      \"endTime\": \"2026-10-27T17:22:33.122027279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-czkfqz2c2dgm--managed/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-czkfqz2c2dgm--managed/providers/Microsoft.Network/networkInterfaces/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n            correlation ID: 7178816c-96c8-477b-bc2c-2cc5e707ac42\n            client request ID: 3fb94b9b-1ccb-4142-9d30-c007dbf5afcb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x141a49f31000\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-czkfqz2c2dgm\" to finish creating: GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"name\": \"8e112abb-be59-4bbd-9c82-e954bb3d2364\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T17:03:30.992910534Z\",\n      \"endTime\": \"2026-10-27T17:22:33.122027279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-czkfqz2c2dgm--managed/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-czkfqz2c2dgm--managed/providers/Microsoft.Network/networkInterfaces/u8s1d9e0n9d6r0t-autoscale-az-rvm97-74svb-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5092608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e112abb-be59-4bbd-9c82-e954bb3d2364\n            correlation ID: 7178816c-96c8-477b-bc2c-2cc5e707ac42\n            client request ID: 3fb94b9b-1ccb-4142-9d30-c007dbf5afcb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112256785092608","occurred_at":"2026-10-27T16:03:40Z","signature_id":"f9f25242b172db559f6d87423a34b4f0d1ff324276218bc210554fe6763ef622","pr_number":7730,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e0c9b2f5fa0da8e29aa067a0a42cc3a25fadc4fe77c282ebc769212a8b5a7957","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d7d76a6b60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n574rs4kgzmb\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"name\": \"0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:22.647294159Z\",\n      \"endTime\": \"2026-10-26T04:05:25.39660378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n            correlation ID: 2461ce1f-a2e3-4c70-ae70-58c7bc6e13e3\n            client request ID: 87ad1faa-4a8b-48cb-888d-882999d849fb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d7d76a6b60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n574rs4kgzmb\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"name\": \"0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:22.647294159Z\",\n      \"endTime\": \"2026-10-26T04:05:25.39660378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n            correlation ID: 2461ce1f-a2e3-4c70-ae70-58c7bc6e13e3\n            client request ID: 87ad1faa-4a8b-48cb-888d-882999d849fb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","signature_id":"f5c9cc123de6b12a7bef437313c3b38c0c01be58bd0124c1453666401f1de8ac","pr_number":7721,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df518282228b252a66b7fa05de10041466f90edc2b99ca2d14dd0565ca45d5ae","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zgspqd\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1120db708700\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-zgspqd: failed waiting for cluster=\"cluster-zstream-4-22-zgspqd\" in resourcegroup=\"rg-zstream-upgrade-4-22-4qv6w5xvr977\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\",\n      \"name\": \"ffe95342-e02c-4496-883e-5b1bf611d99e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:19.965849634Z\",\n      \"endTime\": \"2026-10-25T11:30:25.598823992Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, olm-operator, capi-provider, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, cluster-storage-operator, catalog-operator, openshift-controller-manager, cluster-version-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n            correlation ID: e700165f-ec9f-48e4-abf2-18b14bbd9d95\n            client request ID: cdbafbf0-d5e0-4ed7-8b28-b50e4bf8f6b8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zgspqd\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1120db708700\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-zgspqd: failed waiting for cluster=\"cluster-zstream-4-22-zgspqd\" in resourcegroup=\"rg-zstream-upgrade-4-22-4qv6w5xvr977\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\",\n      \"name\": \"ffe95342-e02c-4496-883e-5b1bf611d99e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:19.965849634Z\",\n      \"endTime\": \"2026-10-25T11:30:25.598823992Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, olm-operator, capi-provider, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, redhat-operators-catalog, openshift-route-controller-manager, community-operators-catalog, certified-operators-catalog, cluster-storage-operator, catalog-operator, openshift-controller-manager, cluster-version-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffe95342-e02c-4496-883e-5b1bf611d99e\n            correlation ID: e700165f-ec9f-48e4-abf2-18b14bbd9d95\n            client request ID: cdbafbf0-d5e0-4ed7-8b28-b50e4bf8f6b8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"6007f17af9a420d188821078ddfcd13ef9adc03b75055acbd2a1cc8c5fb10108","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e1308aaf3b1d1878569e7faaa357169a3760ff01fb807dd5a0044acf0491d40b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x105dbf764180\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mv5sq4l7jv4r\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n      \"name\": \"35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:21.658859222Z\",\n      \"endTime\": \"2026-10-25T11:30:25.924003842Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: capi-provider; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n            correlation ID: d887bb88-655e-4cf4-84f8-c59ec1d133f4\n            client request ID: 066a4e25-cc0e-4d6f-a9c5-8e71913c3b6c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x105dbf764180\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-mv5sq4l7jv4r\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n      \"name\": \"35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:21.658859222Z\",\n      \"endTime\": \"2026-10-25T11:30:25.924003842Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: capi-provider; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/35ffd4e9-2ccb-4812-b71c-6d6a35fe8b0b\n            correlation ID: d887bb88-655e-4cf4-84f8-c59ec1d133f4\n            client request ID: 066a4e25-cc0e-4d6f-a9c5-8e71913c3b6c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"8c3d03bb192574d4a7d4caa279bb2bc24cd1fc9e953bcb7a6725c28dc34a8baa","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"387b7a3bffcaeae5fc4f0b57ac52dafe4294f605cab90681b8f7faa90ebf4b9b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1afe2fabc320\u003e: \n    failed to create HCP cluster ea-list7tqb74: failed waiting for cluster=\"ea-list7tqb74\" in resourcegroup=\"ea-list-8mbx8mstb22c\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\",\n      \"name\": \"506cd809-e6bc-45d8-aa3a-45042c374b41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:00.137646561Z\",\n      \"endTime\": \"2026-10-25T11:31:06.609068276Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, community-operators-catalog, packageserver, cluster-node-tuning-operator, cluster-storage-operator, ingress-operator, dns-operator, catalog-operator, cluster-network-operator, cluster-version-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, cluster-policy-controller, olm-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n            correlation ID: a86e7a8c-dce2-49be-8312-2dfeeebedc58\n            client request ID: 03f569a8-daf7-4041-b81b-1efe61e6782e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1afe2fabc320\u003e: \n    failed to create HCP cluster ea-list7tqb74: failed waiting for cluster=\"ea-list7tqb74\" in resourcegroup=\"ea-list-8mbx8mstb22c\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\",\n      \"name\": \"506cd809-e6bc-45d8-aa3a-45042c374b41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:00.137646561Z\",\n      \"endTime\": \"2026-10-25T11:31:06.609068276Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, community-operators-catalog, packageserver, cluster-node-tuning-operator, cluster-storage-operator, ingress-operator, dns-operator, catalog-operator, cluster-network-operator, cluster-version-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, cluster-policy-controller, olm-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/506cd809-e6bc-45d8-aa3a-45042c374b41\n            correlation ID: a86e7a8c-dce2-49be-8312-2dfeeebedc58\n            client request ID: 03f569a8-daf7-4041-b81b-1efe61e6782e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"35b9f2d778245d855045dd3836882934be33878b78f8ec15263d0fa15d2e6a86","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22535b109360\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bh6sljsqnwrw\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\",\n      \"name\": \"ec3175b8-8979-4854-9d37-2e76e8d09536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:03.316597788Z\",\n      \"endTime\": \"2026-10-25T11:31:06.803246105Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, ingress-operator, cluster-image-registry-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, cluster-node-tuning-operator, control-plane-pki-operator, azure-cloud-controller-manager, openshift-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, packageserver, certified-operators-catalog, catalog-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n            correlation ID: 7d2b87e8-723a-49a9-a4cb-8027a1fb4066\n            client request ID: b5b1bd46-74cb-4267-85b1-e1af986392eb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22535b109360\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-bh6sljsqnwrw\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\",\n      \"name\": \"ec3175b8-8979-4854-9d37-2e76e8d09536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:03.316597788Z\",\n      \"endTime\": \"2026-10-25T11:31:06.803246105Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, ingress-operator, cluster-image-registry-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, cluster-version-operator, cluster-node-tuning-operator, control-plane-pki-operator, azure-cloud-controller-manager, openshift-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, packageserver, certified-operators-catalog, catalog-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec3175b8-8979-4854-9d37-2e76e8d09536\n            correlation ID: 7d2b87e8-723a-49a9-a4cb-8027a1fb4066\n            client request ID: b5b1bd46-74cb-4267-85b1-e1af986392eb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"774ef136f38611a7eefcb40f8fdd5f5fd82c4b8dae09106b8507e7a803347d4e","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8587d6ef8cd7dac5fe4005b92f7045c941c29895bdc330fc36d748cf48c16a17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T17:04:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n      \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n      \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n            correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n            client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n      \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n      \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n            correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n            client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"3cd718d00c36786db9c07977d8d9bab6da7e3aa088465ca79459dcdaa748a25b","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0316cbd90cb1deb59d43692438e3753b878c1a8c6b4dfbaa4805ffd81627a594","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cab81a88220\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-kfvwp2t79pvw\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"name\": \"b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:43.851254184Z\",\n      \"endTime\": \"2026-10-26T04:05:45.382791214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n            correlation ID: 63ad8909-3abd-481b-ad9d-99ee8f5ef24a\n            client request ID: 2dbc76d4-ed2f-44e9-8bf3-81fac3665977\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cab81a88220\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-kfvwp2t79pvw\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"name\": \"b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:43.851254184Z\",\n      \"endTime\": \"2026-10-26T04:05:45.382791214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n            correlation ID: 63ad8909-3abd-481b-ad9d-99ee8f5ef24a\n            client request ID: 2dbc76d4-ed2f-44e9-8bf3-81fac3665977\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","signature_id":"f42b397030cfd80714e248b2e32faebf18a0825358e6552e7da3dfc682076e7f","pr_number":7721,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"eaaeea36f37284592eadff855e2786beb007e4407cc639524af8e920f7d265fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-vfvmsnwsd5sg/cluster-candidate-5-0-lh7h4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x274288768300\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-lh7h4v: failed waiting for cluster=\"cluster-candidate-5-0-lh7h4v\" in resourcegroup=\"rg-candidate-5-0-vfvmsnwsd5sg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n      \"name\": \"f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:22.79701272Z\",\n      \"endTime\": \"2026-10-25T11:30:25.820223913Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 20m56s ago; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n            correlation ID: d572d016-0277-4113-8a96-1da300eb89da\n            client request ID: a981c6e6-813e-4373-81f7-4aab2cf9a886\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-5-0-vfvmsnwsd5sg/cluster-candidate-5-0-lh7h4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x274288768300\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-lh7h4v: failed waiting for cluster=\"cluster-candidate-5-0-lh7h4v\" in resourcegroup=\"rg-candidate-5-0-vfvmsnwsd5sg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n      \"name\": \"f8664d31-ed00-44b7-89f1-f9abec4ea6b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:22.79701272Z\",\n      \"endTime\": \"2026-10-25T11:30:25.820223913Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 20m56s ago; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f8664d31-ed00-44b7-89f1-f9abec4ea6b8\n            correlation ID: d572d016-0277-4113-8a96-1da300eb89da\n            client request ID: a981c6e6-813e-4373-81f7-4aab2cf9a886\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"dff12b42279b00707352b18af2f0050316e2032026e4d08cb488796c14d37bc7","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6b9eb312202879fb9a6f390df391c95e64889c51047a74b640051be8935c181f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a8a07a1e540\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rvs4gzxj9lgg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\",\n      \"name\": \"63403578-fb62-456b-b40e-41ef79e2e399\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:39.014043295Z\",\n      \"endTime\": \"2026-10-25T11:31:45.790547256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m13s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n            correlation ID: 38478f64-7ba7-40aa-b720-a7f94d356d8b\n            client request ID: 82836f4c-341e-4d1e-88f4-c55b4e681a30\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a8a07a1e540\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rvs4gzxj9lgg\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\",\n      \"name\": \"63403578-fb62-456b-b40e-41ef79e2e399\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:02:39.014043295Z\",\n      \"endTime\": \"2026-10-25T11:31:45.790547256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m13s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63403578-fb62-456b-b40e-41ef79e2e399\n            correlation ID: 38478f64-7ba7-40aa-b720-a7f94d356d8b\n            client request ID: 82836f4c-341e-4d1e-88f4-c55b4e681a30\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"dcd430427d782eeea86e74cee94b16d2a3435cb845ddf1163daad51281789a78","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"663e8d8eeb1c1b6a6d00064673e9a3330c994712d05117a165f749566b26a621","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a59ed8de2a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-xf7h6gdhg87b\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n      \"name\": \"0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:20.835629052Z\",\n      \"endTime\": \"2026-10-25T11:30:25.907970891Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n            correlation ID: f5a00756-4d2f-4428-b3ea-e6cf5e3ef0a1\n            client request ID: 03254b25-a5c1-4e3b-88dd-5aeb8bec557c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:80]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a59ed8de2a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-xf7h6gdhg87b\" to finish creating: GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n      \"name\": \"0c13cc8b-efe6-4584-9ae2-571f7f85e94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T11:01:20.835629052Z\",\n      \"endTime\": \"2026-10-25T11:30:25.907970891Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c13cc8b-efe6-4584-9ae2-571f7f85e94b\n            correlation ID: f5a00756-4d2f-4428-b3ea-e6cf5e3ef0a1\n            client request ID: 03254b25-a5c1-4e3b-88dd-5aeb8bec557c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"5d52aeb2aaff4eeba7d8d90ff684f12b2a5726d50ed448947775022cdfac17e6","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea06060b15ab7907c4f534ea9c38687a98d66cbc48379a8dd476993e78886883","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26678ea60858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f8mftsvbkmbp\" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"name\": \"95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:20:09.676863434Z\",\n      \"endTime\": \"2026-10-27T01:39:10.703576848Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a\n            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26678ea60858\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f8mftsvbkmbp\" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"name\": \"95ed2027-596a-4016-be69-40cc2e92cf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T01:20:09.676863434Z\",\n      \"endTime\": \"2026-10-27T01:39:10.703576848Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f\n            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a\n            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114870474402959360","occurred_at":"2026-10-27T00:03:08Z","signature_id":"171fe6317ac5031415995a5b49fd173b8d76345e4653ed4a832f67862f6d0e2b","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2913b68eb188eee155973da8f2278b815824c08e26b8568b029df43566c7bce9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T08:17:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n      \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n      \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n            correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n            client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n      \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n      \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n            correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n            client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"f9315d1a908c869487b8528009d6e3be71e485ce7d3c349af0d4312ee843c244","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-23T08:32:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n      \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n      \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n            correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n            client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n      \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n      \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n            correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n            client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7708/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113549009753739264","occurred_at":"2026-10-23T08:32:06Z","signature_id":"504e0ef2def37e129dc9af51f3cea8c5643dc0df0434eba4232abc7a90d05a61","pr_number":7708,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2bd348e2f759f173403310693a20ab58f53f43ab8426ca2e577a35357f75f147","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; hypershift NodePool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T15:20:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xed2ec9e01c0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-989gmc\" resourcegroup=\"rg-np-version-upgrade-989gmc-8hflngjf7pjq\" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\",\n      \"name\": \"33eeff39-7b04-4af1-8912-9297ff607fdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T16:42:14.996433941Z\",\n      \"endTime\": \"2026-10-22T17:26:16.905655326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n            correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce\n            client request ID: 143bee29-fe61-4aa3-b798-569832148329\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xed2ec9e01c0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-989gmc\" resourcegroup=\"rg-np-version-upgrade-989gmc-8hflngjf7pjq\" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\",\n      \"name\": \"33eeff39-7b04-4af1-8912-9297ff607fdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T16:42:14.996433941Z\",\n      \"endTime\": \"2026-10-22T17:26:16.905655326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb\n            correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce\n            client request ID: 143bee29-fe61-4aa3-b798-569832148329\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289616629960704","occurred_at":"2026-10-22T15:20:57Z","signature_id":"fe24240d8784318f59cd835bc925e1e72ca0ec51e04dd27b76e1c9a75ad9c2f0","pr_number":7646,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x240d16aa66e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-z9s25h\" in resourcegroup=\"arm64-vm-cluster-r2854rq2nbt4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n            correlation ID: 23c397c2-e55d-434d-a806-434e6889bce6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x240d16aa66e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-z9s25h\" in resourcegroup=\"arm64-vm-cluster-r2854rq2nbt4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n            correlation ID: 23c397c2-e55d-434d-a806-434e6889bce6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114635485509849088","occurred_at":"2026-10-26T08:29:22Z","signature_id":"541b77824c3f3a313c9ef1c117ae9a8f71c935bf6deaab5e9c18791b44825d21","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4e9cd130a5a7f9bb66c6119ae4848ee76685bc008fdd6564ac11ea576600fa92","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/\u003cuuid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T20:25:38Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-22T21:04:17.903Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-22T21:04:18.672Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-22T21:04:19.255Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3800704-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e\ntime=2026-10-22T21:07:49.884Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e/operationStatuses/08584089038265307890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/fe73d360-7d74-562a-b1ee-44865af0a11a' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/22/2026 9:06:20 PM' with correlationId 'fe6d7e33-2867-4647-8fe8-d9c4bbb9af94', and will expire at '10/29/2026 9:06:20 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":7703,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c11ecd2e0a07f870d594200c753517b1b1da13ef27c39ff3dad4af7f1e59078f","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-27T21:04:40.560Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-27T21:04:41.167Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-27T21:04:41.720Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6348544-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0\ntime=2026-10-27T21:12:43.185Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6348544-mgmt-1/providers/Microsoft.Resources/deployments/07716c56beddc7b7764d03db8be5e4423d70cd91036a0e2812ce90dae36950e0/operationStatuses/08584084718040326968\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6348544-mgmt-1/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/27/2026 9:05:02 PM' with correlationId '91b74ae1-3277-41f6-8cd7-59f2d0861c2c', and will expire at '11/3/2026 9:05:02 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115175058346348544","occurred_at":"2026-10-27T20:13:18Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":7441,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0003cc8ca3a9f7347553bcc0638ed02c32204aa3c9fd097c1f10d2bf5392fbd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T17:04:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n            correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n            correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"86725f75fec081c22e7a197ddb0df251563b2a69bc88d75ade0edf6269655c72","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-23-4q7sfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1abe3e2ce320\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2\" in resourcegroup=\"rg-zstream-upgrade-4-23-r8wjcwsdhsgm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n            correlation ID: adb21106-8945-4c8c-8718-f7a2e847f630\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-23-4q7sfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1abe3e2ce320\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2\" in resourcegroup=\"rg-zstream-upgrade-4-23-r8wjcwsdhsgm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-23-r8wjcwsdhsgm/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-4-23-4q7sfm-ffc6p2/operationStatuses/08584085034054152549\n            correlation ID: adb21106-8945-4c8c-8718-f7a2e847f630\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115039807775182848","occurred_at":"2026-10-27T11:16:00Z","signature_id":"09bf758da913ed0a3cc99ca9f1bd9cf87fbe7bfffcfae83211bf1e062a809761","pr_number":7768,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8199ed0284e70ecfb2dc70637f49d2db3d9241a37d6a24ae7dcc8841eac024f4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidRequest; detail message Call to Microsoft.EventGrid/namespaces failed. Error message: Required property 'linkIdentifier' not found in JSON. Path 'remotePrivateEndpoint.connectionDetails[0]', line 1, position 932.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-27T07:31:38.140Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-27T07:31:38.545Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-27T07:31:39.254Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1948032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0\ntime=2026-10-27T07:33:09.610Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0/operationStatuses/08584085205866172617\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Call to Microsoft.EventGrid/namespaces failed. Error message: Required property 'linkIdentifier' not found in JSON. Path 'remotePrivateEndpoint.connectionDetails[0]', line 1, position 932.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": []\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114973576971948032","occurred_at":"2026-10-27T06:52:49Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T12:05:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-pfvhlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e040752480\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hkoaxxduqas2k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Authorization/roleAssignments/1695ac58-5845-59e4-be60-94380c7b782a' at line '1' and column '21698'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 21698,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n            correlation ID: c39b7be0-22bb-49c9-96d8-f53c5e4ad2fd\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-pfvhlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e040752480\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hkoaxxduqas2k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqpxn7gcfmxn/providers/Microsoft.Authorization/roleAssignments/1695ac58-5845-59e4-be60-94380c7b782a' at line '1' and column '21698'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 21698,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-pfvhlb-4rsh8t/operationStatuses/08584084473369857031\n            correlation ID: c39b7be0-22bb-49c9-96d8-f53c5e4ad2fd\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115268754764992512","occurred_at":"2026-10-28T02:25:45Z","signature_id":"1743e95f5bf74e9cfa2904f7bbd78e3557a8af048b321b3e791e03d323b382a0","pr_number":7791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-12","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2401cda17700\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-6lgfpz\" in resourcegroup=\"rg-np-labels-taints-2pq6dbldqrjk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: f50cf33f-8bc4-4ce0-8a7c-d816cfc4e960\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-bvffjvcy6qo3o' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n            correlation ID: 9ef1eaf3-8ba8-425e-8fe5-70f8311af59e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2401cda17700\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-6lgfpz\" in resourcegroup=\"rg-np-labels-taints-2pq6dbldqrjk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details. Correlation ID: f50cf33f-8bc4-4ce0-8a7c-d816cfc4e960\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-bvffjvcy6qo3o' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-2pq6dbldqrjk/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-6lgfpz/operationStatuses/08584086797384633997\n            correlation ID: 9ef1eaf3-8ba8-425e-8fe5-70f8311af59e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"3d343371079ea4c8eec5fdc9bbbf22e9c2c37580483fe6a2b69023d256138a61","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T12:57:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2ed9255f4018\u003e: \n    resource group \"rg-candidate-4-22-f565h2jdmqgh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-4j4s8h\" in resourcegroup=\"rg-candidate-4-22-f565h2jdmqgh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n      \"name\": \"837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:27:18.140344478Z\",\n      \"endTime\": \"2026-10-22T14:51:19.380950943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n            correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489\n            client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2ed9255f4018\u003e: \n    resource group \"rg-candidate-4-22-f565h2jdmqgh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-4j4s8h\" in resourcegroup=\"rg-candidate-4-22-f565h2jdmqgh\" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n      \"name\": \"837480d6-dd8d-4baa-bb95-0329fd032c3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:27:18.140344478Z\",\n      \"endTime\": \"2026-10-22T14:51:19.380950943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c\n            correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489\n            client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7686/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113253511574589440","occurred_at":"2026-10-22T12:57:54Z","signature_id":"9ee611bb2a3e02c3800fcd63a0f4ec2bc6c1c6da5f8de8fd0a89b2b935289a9e","pr_number":7686,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f507da136c9daa16f6c852638ab5c837615df7fc7163e85f3926a57d8414cb44","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-4sk7fq' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-jz7tkx\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3490511d0108\u003e: \n    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record \"*.e2e-dns-4sk7fq\" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-jz7tkx\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3490511d0108\u003e: \n    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record \"*.e2e-dns-4sk7fq\" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq\n            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","signature_id":"61541f3b5ca8921faa4b85328182f5e428d4643d60841d69d788f32ca5efe589","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c68cfdadf7a813f7306833771fe3f1479e7865cf15d87626b36ad658cb57e3bf","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-bkpxxt' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-7tbfp9\"\nUnexpected error:\n    \u003c*errors.joinError | 0x28672532e420\u003e: \n    VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-bkpxxt\" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n            correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-7tbfp9\"\nUnexpected error:\n    \u003c*errors.joinError | 0x28672532e420\u003e: \n    VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-bkpxxt\" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt\n            correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115533269758906368","occurred_at":"2026-10-28T19:56:50Z","signature_id":"0e25a1ce387e1311f273f459aa16bea882049776f82a3cf61fd90170dc6dacb5","pr_number":7757,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b703f4204991a61d0cf90f658410c32ca25e0cbc0d2e9af5a53fcc8458b106b4","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-czbqgt' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-kzwwft\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2fea822a5d0\u003e: \n    VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record \"*.e2e-dns-czbqgt\" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n            correlation ID: ef507981-8069-4371-b52a-e18fb184cf37\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-kzwwft\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2fea822a5d0\u003e: \n    VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record \"*.e2e-dns-czbqgt\" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt\n            correlation ID: ef507981-8069-4371-b52a-e18fb184cf37\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115509692493991936","occurred_at":"2026-10-28T18:23:09Z","signature_id":"430a37f54fa88a8257498cad29293dfb1d7b7872cf2fdd8d9629d86a48de04bf","pr_number":7785,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e2f2ef9a0323545de216b10f60b08cc70ff2c858dbbd51d763ae3430b634070","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-fs2vtm' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-9pvn6m\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6c70d0280d8\u003e: \n    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record \"*.e2e-dns-fs2vtm\" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-9pvn6m\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6c70d0280d8\u003e: \n    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record \"*.e2e-dns-fs2vtm\" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm\n            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115052847685963776","occurred_at":"2026-10-27T12:07:44Z","signature_id":"709a8c51e125e721b929c34d4ce6d33c0724708a7660f16ba59386319e0d5760","pr_number":7373,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6bfef0b8bc199ef07e1bd5837228e606b5ba41f12b4a928e0e7b19c9c7610dda","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-gkfmqf' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-n5f5ld\"\nUnexpected error:\n    \u003c*errors.joinError | 0x29f2244004b0\u003e: \n    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-gkfmqf\" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-n5f5ld\"\nUnexpected error:\n    \u003c*errors.joinError | 0x29f2244004b0\u003e: \n    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-gkfmqf\" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf\n            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115207073061933056","occurred_at":"2026-10-27T22:20:39Z","signature_id":"c3b990384c16e31bb3ad5ff9f7b85816b1f59b27a4fe9f73c57e1f5d44e83c84","pr_number":6924,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"19089bef7ce831b1ccaf302c1307f8a54f6b9033225b28b6d11d82d8f8c236be","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-lpggdd' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-tdqwb6\"\nUnexpected error:\n    \u003c*errors.joinError | 0x122a895e0150\u003e: \n    VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) timed out after 10m0.001s: DNS A record \"*.e2e-dns-lpggdd\" not yet present in zone rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-lpggdd' does not exist in resource group 'rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n            correlation ID: fc55c026-0ddc-4f29-bb3d-f393fa0f3cd6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-tdqwb6\"\nUnexpected error:\n    \u003c*errors.joinError | 0x122a895e0150\u003e: \n    VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud) timed out after 10m0.001s: DNS A record \"*.e2e-dns-lpggdd\" not yet present in zone rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-lpggdd' does not exist in resource group 'rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qhznhv86fzbg-candidate-tdqwb6--managed/providers/Microsoft.Network/dnsZones/aro.r1n2g0k6g2s3t3a.soq5.j8963712.hcp.osadev.cloud/A/*.e2e-dns-lpggdd\n            correlation ID: fc55c026-0ddc-4f29-bb3d-f393fa0f3cd6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115349685118963712","occurred_at":"2026-10-28T07:47:20Z","signature_id":"030be4640c04973776445b68ddb84b084e485fea34e2f96f0c190f92f6f2b41b","pr_number":7786,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7bc5b7efd9282638ed5ca12c2caee514f8621db6c432518ebabc0a7a96fa5799","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-p2v629' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-r5cltn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263db2a8cd50\u003e: \n    VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-p2v629\" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n            correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-r5cltn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263db2a8cd50\u003e: \n    VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-p2v629\" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629\n            correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115500092713603072","occurred_at":"2026-10-28T17:45:00Z","signature_id":"d552ce333ff64238cdb35341cbb435780b950120a489c5c39b9ea39ba2987f7f","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a82463256e75ac51402f90e1a076775c96cdeed8daf25e578b182ea9407bbbf8","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-qg48vc' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-458kh7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1b0cdceb6450\u003e: \n    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-qg48vc\" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-458kh7\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1b0cdceb6450\u003e: \n    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record \"*.e2e-dns-qg48vc\" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc\n            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","signature_id":"54722d0e8af7431db3f910c6b8799d624cfe0c8f4b5fa53109f9ee0b900c8d0d","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4a04fe062c510b6c532fad92f851f6d9fcd4a47e168844e52656b2097c3939c","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-rdkvkj' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T23:07:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cprxqg\"\nUnexpected error:\n    \u003c*errors.joinError | 0x30c853d502b8\u003e: \n    VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-rdkvkj\" not yet present in zone rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rdkvkj' does not exist in resource group 'rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n            correlation ID: 984d9387-98d9-461e-8f5b-657273ddef57\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cprxqg\"\nUnexpected error:\n    \u003c*errors.joinError | 0x30c853d502b8\u003e: \n    VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-rdkvkj\" not yet present in zone rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rdkvkj' does not exist in resource group 'rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hzrgxmfgtllm-candidate-cprxqg--managed/providers/Microsoft.Network/dnsZones/aro.n9p8g3s6f9q9f8h.s668.j0574848.hcp.osadev.cloud/A/*.e2e-dns-rdkvkj\n            correlation ID: 984d9387-98d9-461e-8f5b-657273ddef57\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114494146650574848","occurred_at":"2026-10-25T23:07:44Z","signature_id":"ecf89ec70a260529697f24e15845fd8acfb2ceebcc45bfb36b26f438351b97bf","pr_number":7739,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a9db3befe442ad19ff8afe1f17b9e1c81ebea1992a9802344e12294deb314074","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-swrng7' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-v9km2q\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1f131b624198\u003e: \n    VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-swrng7\" not yet present in zone rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-swrng7' does not exist in resource group 'rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n            correlation ID: f5ba2020-eed4-41d4-be4e-2d7ff36da168\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-v9km2q\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1f131b624198\u003e: \n    VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record \"*.e2e-dns-swrng7\" not yet present in zone rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-swrng7' does not exist in resource group 'rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-2f2k8zzmfdnf-candidate-v9km2q--managed/providers/Microsoft.Network/dnsZones/aro.r6b2z1c7z7d0i9t.kvy2.j8974848.hcp.osadev.cloud/A/*.e2e-dns-swrng7\n            correlation ID: f5ba2020-eed4-41d4-be4e-2d7ff36da168\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401780928974848","occurred_at":"2026-10-28T11:14:07Z","signature_id":"20463c5543b2bf203bbfa488866984362e0c3b8eb919f697fb8074e808e6a4f1","pr_number":7786,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-22T01:17:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-5h8rfd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d0abaa07220\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster=\"cluster-zstream-4-23-5h8rfd\" in resourcegroup=\"rg-zstream-upgrade-4-23-5tgvrgshsr5n\" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n            correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb\n            client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-5h8rfd\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d0abaa07220\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster=\"cluster-zstream-4-23-5h8rfd\" in resourcegroup=\"rg-zstream-upgrade-4-23-5tgvrgshsr5n\" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85\n            correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb\n            client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113077350928224256","occurred_at":"2026-10-22T01:17:54Z","signature_id":"e671cab1177a0493b74cb326ec698815954811c1abba3003abc61755f7f3f93b","pr_number":6128,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"73cc879e56f570416795adcce7e08e28387abc22a0432f0cca452338bd4aff84","failure_pattern":"Get \"\u003curl\u003e\": context deadline exceeded","search_query":"Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-15T23:56:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.007s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0xc2056a72b10\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.007s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\nUnexpected error:\n    \u003c*url.Error | 0xc2056a72b10\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\": context deadline exceeded\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.qjr7.j5092608.hcp.osadev.cloud\",\n        Err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115112256785092608","occurred_at":"2026-10-27T16:03:40Z","signature_id":"c6ac03d3d873d7b532f5a92eba3e5d864ae3c62d4f24763727d7aa7fc6ab9621","pr_number":7730,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T21:36:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","signature_id":"ffad9075f8b073e455e1ac724969c29ae57f1247a7dc73dc4b06d2983015219a","pr_number":7733,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8d06c33c9ca8da47b85ff204ce768771e156eaca59fdc3f367974352e4e17539","failure_pattern":"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod \u003cnamespace\u003e/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-20-csbqrn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x395999ae65e8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39599a424e20\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0x395999e53a40\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0x39599931d360\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-20-csbqrn\"\nUnexpected error:\n    \u003c*errors.joinError | 0x395999ae65e8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39599a424e20\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0x395999e53a40\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-disk-6p6t4/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0x39599931d360\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115118404208693248","occurred_at":"2026-10-27T16:28:19Z","signature_id":"adcabdd64318a6bbc42056fc4a3695e48135925c38c3cccf257c171d24c1fa4c","pr_number":7495,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0c9a69ce334c27525179441c4e4ad72c2ac5b4078f88ec831ce8d15009143a74","failure_pattern":"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod \u003cnamespace\u003e/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-sbw4s5\"\nUnexpected error:\n    \u003c*errors.joinError | 0xae6c2dd4150\u003e: \n    VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xae6c30c2320\u003e{\n                msg: \"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0xae6c3360200\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0xae6c335a000\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-sbw4s5\"\nUnexpected error:\n    \u003c*errors.joinError | 0xae6c2dd4150\u003e: \n    VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\"unknown\") has prevented the request from succeeding (get pods e2e-csi-pod)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xae6c30c2320\u003e{\n                msg: \"VerifyCSIPersistentVolume(file.csi.azure.com) failed: failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                err: \u003c*fmt.wrapError | 0xae6c3360200\u003e{\n                    msg: \"failed to get logs for pod e2e-csi-file-tmd2m/e2e-csi-pod after it succeeded: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                    err: \u003c*errors.StatusError | 0xae6c335a000\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods e2e-csi-pod)\",\n                            Reason: \"InternalError\",\n                            Details: {\n                                Name: \"e2e-csi-pod\",\n                                Group: \"\",\n                                Kind: \"pods\",\n                                UID: \"\",\n                                Causes: [\n                                    {\n                                        Type: \"UnexpectedServerResponse\",\n                                        Message: \"unknown\",\n                                        Field: \"\",\n                                    },\n                                ],\n                                RetryAfterSeconds: 0,\n                            },\n                            Code: 500,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115082291888787456","occurred_at":"2026-10-27T14:04:49Z","signature_id":"32ca328ce4f4d1f24d28c583c2c75a25c3a62e05b6f6a593c0daa6efba4fd145","pr_number":7770,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"057c32882f63a3eb320e4cfbfa6ca35e12ecb53508e538854f6156c738380877","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node cluster-np-prdbsm-\u003cid\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-\u003cid\u003e\": nodes \"cluster-np-prdbsm-\u003cid\u003e\" is forbidden","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-prdbsm\"\nUnexpected error:\n    \u003c*errors.joinError | 0xfaf21d3c618\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xfaf223bc0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0xfaf223bc080\u003e{\n                    msg: \"failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0xfaf223bc060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0xfaf21fc43c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"cluster-np-prdbsm-dxj9f-p58c2\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-prdbsm\"\nUnexpected error:\n    \u003c*errors.joinError | 0xfaf21d3c618\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xfaf223bc0a0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0xfaf223bc080\u003e{\n                    msg: \"failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0xfaf223bc060\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"cluster-np-prdbsm-dxj9f-p58c2\\\": nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0xfaf21fc43c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"cluster-np-prdbsm-dxj9f-p58c2\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"cluster-np-prdbsm-dxj9f-p58c2\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115208311295971328","occurred_at":"2026-10-27T22:25:34Z","signature_id":"8febf95c3a0ca16f0a2a3359fd8ca2fcc7ba552e29fd5c4139ff44661fb4f3d7","pr_number":7787,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9b156542f61a22d88e8e787cb5184aae92eaf130a724dfa3a85c4427efdde52","failure_pattern":"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T19:19:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n            observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n            observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113711841971539968","occurred_at":"2026-10-23T19:19:08Z","signature_id":"06405e4c71e72b70733326b19524a23a628ce5d6d98f1e230f7156ce01e4dc2e","pr_number":7727,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ec19e62203173fac0d4f8045f354fdbd5a551b968f9125422e7f270d533b0922","failure_pattern":"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-546c5c\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e4bde720b40\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x2e4bde720ac0\u003e{\n            observation: \u003c*errors.errorString | 0x2e4be03290f0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2e4bde720aa0\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-546c5c\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e4bde720b40\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x2e4bde720ac0\u003e{\n            observation: \u003c*errors.errorString | 0x2e4be03290f0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2e4bde720aa0\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590084358475776","occurred_at":"2026-10-26T05:28:57Z","signature_id":"cb667908309974df41a36846f12e4ef39e2292926044076fc3350fbe6a737f00","pr_number":7727,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aee4a820e8bbfea9c5305e481cde65284da0e2c801d92f6a858dc46e820847ed","failure_pattern":"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"\u003cuuid\u003e\": Get \"\u003curl\u003e\": can't get access token: invalid_grant: Invalid refresh token","search_query":"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f4ce1090c80\u003e: \n    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token\n    {\n        msg: \"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n        err: \u003c*fmt.wrapError | 0x1f4ce110ccc0\u003e{\n            msg: \"failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n            err: \u003c*url.Error | 0x1f4ce0ce8a80\u003e{\n                Op: \"Get\",\n                URL: \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\",\n                Err: \u003c*fmt.wrapError | 0x1f4ce110cca0\u003e{\n                    msg: \"can't get access token: invalid_grant: Invalid refresh token\",\n                    err: \u003c*errors.errorString | 0x1f4ce1105790\u003e{\n                        s: \"invalid_grant: Invalid refresh token\",\n                    },\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f4ce1090c80\u003e: \n    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\": can't get access token: invalid_grant: Invalid refresh token\n    {\n        msg: \"VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n        err: \u003c*fmt.wrapError | 0x1f4ce110ccc0\u003e{\n            msg: \"failed to query OCM clusters API for external_id \\\"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\\\": Get \\\"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\\\": can't get access token: invalid_grant: Invalid refresh token\",\n            err: \u003c*url.Error | 0x1f4ce0ce8a80\u003e{\n                Op: \"Get\",\n                URL: \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27\u0026size=1\",\n                Err: \u003c*fmt.wrapError | 0x1f4ce110cca0\u003e{\n                    msg: \"can't get access token: invalid_grant: Invalid refresh token\",\n                    err: \u003c*errors.errorString | 0x1f4ce1105790\u003e{\n                        s: \"invalid_grant: Invalid refresh token\",\n                    },\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115097821592948736","occurred_at":"2026-10-27T15:06:31Z","signature_id":"7e1c250216db8719c65e193ed331feca05376cad3f3ee5d0c571c8845c69f894","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2f3aceed2bd993d066f5fc126ddd12907122d755568207fafbd6490e3839a54a","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T05:21:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    {\n        msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n        err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n            s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    {\n        msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n        err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n            s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8c2d12315219ca1579f718de9c20b9b687edb43eb1f14b0f2771853e17c295f1","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df10c7ad9b6f58b98b74568dc817b34eae1f531ef761f38ba689e30339ed72f4","failure_pattern":"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa7745440900\u003e: \n    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()\n    {\n        msg: \"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n        err: \u003c*verifiers.pollObservationError | 0xa77454408e0\u003e{\n            observation: \u003c*errors.errorString | 0xa77456c4580\u003e{\n                s: \"ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xa77454408c0\u003e{\n                msg: \"failed to get ClusterOperator \\\"insights\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa7745440900\u003e: \n    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()\n    {\n        msg: \"VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n        err: \u003c*verifiers.pollObservationError | 0xa77454408e0\u003e{\n            observation: \u003c*errors.errorString | 0xa77456c4580\u003e{\n                s: \"ClusterOperator \\\"insights\\\" has no Disabled condition yet ()\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xa77454408c0\u003e{\n                msg: \"failed to get ClusterOperator \\\"insights\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115043139386347520","occurred_at":"2026-10-27T11:29:14Z","signature_id":"d7f51ef3f79440dd76467c8a42d472c3bd9858d482536084ca3eb59fd60d6127","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50aafdd0364e960cb215062171251b48e16daefd02feb64915b33b81ee7aa50a","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T11:26:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n            observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n                s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x1df679343380\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n            observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n                s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x1df679343380\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"7bb7aa38d25f69b831b53681016019396c025b2d18da7932ffb9e298e6579b79","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a6e167b4896a6cc8e5dc9db9cc3f6643b41874460ed57be902ccc03e9ff1333","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T04:30:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x286c46f305a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x286c46f30580\u003e{\n            observation: \u003c*errors.errorString | 0x286c458cdbb0\u003e{\n                s: \"dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x286c46f30560\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x286c4628c1e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x286c46f30540\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x286c478a50e0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x286c46f305a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x286c46f30580\u003e{\n            observation: \u003c*errors.errorString | 0x286c458cdbb0\u003e{\n                s: \"dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x286c46f30560\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x286c4628c1e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x286c46f30540\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x286c478a50e0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850684494319616","occurred_at":"2026-10-24T04:30:51Z","signature_id":"5577b5c675e94c0cb99a9451504aa28e42c87218c9dfcaaed681e5e942070b2b","pr_number":6772,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a9a6f789dd8ac4b0a8bb332fa94ef06d366fbcb926203ea587fdddca5af97929","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2874138e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2874134f7c90\u003e{\n            s: \"dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2874138e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2874134f7c90\u003e{\n            s: \"dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115008102154637312","occurred_at":"2026-10-27T09:10:01Z","signature_id":"00fd1cc38488aefa9ab60f3c0d51de5fccf6401e991fc65c7e07a02bea28a323","pr_number":7514,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"af2d21e1e88696d05be0ca90beb20ce563a5bf796115e15a05448c96cb57eced","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T21:29:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n            observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n                s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x3481808801e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x3481828e4120\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n            observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n                s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x3481808801e0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x3481828e4120\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","signature_id":"0d5ace1ca3dc48db2d8bb7f3ca58f25eef51b628008732668244b790b4cfd770","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a83a5cad2332d517fe6d82b654adc4ab2b95557c26b786eb1ec2e48ab0c65371","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T06:01:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n            s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n            s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113511186858250240","occurred_at":"2026-10-23T06:01:48Z","signature_id":"8bc9007fc25b5640ebae9f4863d7359c865853bd666601aeb7d286abdb3e9538","pr_number":7653,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d2678f56dc1b3b431d1da623f447c6eafacda9e82db7dae4d918a344f1c056b7","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T00:56:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20a8e1c0eb40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x20a8e1c0eb20\u003e{\n            observation: \u003c*errors.errorString | 0x20a8e1fc7a50\u003e{\n                s: \"dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x20a8e1c0eb00\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x20a8e189aaf0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x20a8e1c0eae0\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x20a8e31d5b60\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20a8e1c0eb40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x20a8e1c0eb20\u003e{\n            observation: \u003c*errors.errorString | 0x20a8e1fc7a50\u003e{\n                s: \"dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x20a8e1c0eb00\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x20a8e189aaf0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x20a8e1c0eae0\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x20a8e31d5b60\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796816469233664","occurred_at":"2026-10-24T00:56:48Z","signature_id":"2782ca05257d433f2f9cf504b4ee91198fa29b5e3ce21b6b86156326f1b22374","pr_number":7738,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6d30b2fd9fdf4368d644d37bdb47abf2d01de4b385c3864e9ebaa6fed07eb9cc","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1623311283c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x1623329ba440\u003e{\n            s: \"dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1623311283c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x1623329ba440\u003e{\n            s: \"dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114552936569769984","occurred_at":"2026-10-26T03:00:55Z","signature_id":"eb67026a69fe8b9ad7c4d8d12e56665ef0ed78588553797d173a254e33ae781b","pr_number":7615,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e56d0b8c2e490620d82d524d28d93e57a3cdbe4dd0316b01ad0ff503a0a27508","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2424493382e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x2424493345b0\u003e{\n            s: \"dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2424493382e0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x2424493345b0\u003e{\n            s: \"dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7809/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115536999510183936","occurred_at":"2026-10-28T20:11:40Z","signature_id":"8507857a28426cbd2659cf220f0916407db048853ec46c983b6852d107a04737","pr_number":7809,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"197f9640ca1cff10bbe506b04538db682332c0dd1edad7858df9b62446a0bf4e","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T17:04:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x794496e4000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x79449964570\u003e{\n            s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x794496e4000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x79449964570\u003e{\n            s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"dfebbf0731f81307daf766e898cda81fae9d9e47bbb68807b280ca90f03dd147","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a7ae0e88d0e3f71d7880a56e396ab0bea1eca0033eece1ef448a2f43eb8cf45","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc252dd22260\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0xc252ebecf50\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc252dd22260\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d5n27jjnnw7n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*errors.errorString | 0xc252ebecf50\u003e{\n            s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7814/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115552146735763456","occurred_at":"2026-10-28T21:11:50Z","signature_id":"d03f0c3a5562bfead821f78003f6870561a945b6d4e25e64c7b3c3ea6ef4ee13","pr_number":7814,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"30002276c0d676e1121ab63d27800e924a0d329b41c2b24beb8a2391f90d92df","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T08:20:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14521ece600\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n            observation: \u003c*errors.errorString | 0x14522933200\u003e{\n                s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x1452193c140\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x14522d0f260\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14521ece600\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n            observation: \u003c*errors.errorString | 0x14522933200\u003e{\n                s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x1452193c140\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x14522d0f260\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","signature_id":"51444fe40b9218630af21f6e92d01bfeda9c0f66e83382df75ec9d39e17d9e40","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"3102a1107f810d1b9d88041d585f2659521f004bd58b18f829fc59f73366743e","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31615201e9a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x31615201e980\u003e{\n            observation: \u003c*errors.errorString | 0x316153080da0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x31615201e960\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x316150f60aa0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x31615201e920\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x316152c10a80\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31615201e9a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x31615201e980\u003e{\n            observation: \u003c*errors.errorString | 0x316153080da0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x31615201e960\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x316150f60aa0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x31615201e920\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9822336-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\\\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x316152c10a80\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115506901079822336","occurred_at":"2026-10-28T18:12:04Z","signature_id":"72129ad72c05e358b350979281ecdf024841a9958c65c7e23f6fa3993b142e2a","pr_number":7704,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4ffb59127307569342e99497be1f78f494e95d4f80f4e55bcb60b61f19b7e0d4","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dc0b3be620\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        err: \u003c*errors.errorString | 0x37dc0b40dbb0\u003e{\n            s: \"dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dc0b3be620\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        err: \u003c*errors.errorString | 0x37dc0b40dbb0\u003e{\n            s: \"dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114843878455513088","occurred_at":"2026-10-26T22:17:27Z","signature_id":"f6e115c9c6b9508390354502456ea7c5a9362dfff4b8c4d0e262e286ea1ff0bb","pr_number":7615,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e61311760ae7623cb7fd4cb9def8b7649d07be11756598455821961566fe69c5","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T16:25:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20c3ca48efe0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x20c3ca48ef80\u003e{\n            observation: \u003c*errors.errorString | 0x20c3cb6d0a90\u003e{\n                s: \"dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x20c3ca48ef60\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x20c3c94b2370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x20c3ca48ef40\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x20c3cb422450\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20c3ca48efe0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*verifiers.pollObservationError | 0x20c3ca48ef80\u003e{\n            observation: \u003c*errors.errorString | 0x20c3cb6d0a90\u003e{\n                s: \"dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x20c3ca48ef60\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x20c3c94b2370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x20c3ca48ef40\u003e{\n                        msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x20c3cb422450\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113305872997289984","occurred_at":"2026-10-22T16:25:57Z","signature_id":"efbf32a99997756a2750ff3841eee191c8cd690170bd70992c2237917b3bee18","pr_number":6823,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"debe837728714debf1279c1ab16e7762e697cbb2740854aa543587193f51a693","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd26a6364d80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0xd26a6364d60\u003e{\n            observation: \u003c*errors.errorString | 0xd26a57e61b0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xd26a6364d40\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0xd26a5820dc0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0xd26a6364d20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0xd26a719e570\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd26a6364d80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0xd26a6364d60\u003e{\n            observation: \u003c*errors.errorString | 0xd26a57e61b0\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xd26a6364d40\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0xd26a5820dc0\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0xd26a6364d20\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j6172544-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\\\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0xd26a719e570\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115401772536172544","occurred_at":"2026-10-28T11:13:52Z","signature_id":"3d9c4de0a857dfc53c34c09e5f993fd7bb77e0bfa2613051219a1203b33d886f","pr_number":7704,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bc8979b405de4c24ab4a1f29e1de7275444b774141612e5f89b0149929ee136c","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3756ccac8820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n        err: \u003c*errors.errorString | 0x3756cb62d010\u003e{\n            s: \"dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3756ccac8820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n        err: \u003c*errors.errorString | 0x3756cb62d010\u003e{\n            s: \"dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7808/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115486053375676416","occurred_at":"2026-10-28T16:49:13Z","signature_id":"b7b55c0bb772493af2f0bf7b377817e6562c148d2c11c583c1c42088a1f89da3","pr_number":7808,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6b19f3be20271a1946ad6e5da34c15339012b8d43c1a7baf3efcab4e3e251f7","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T20:52:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n            s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n            s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","signature_id":"b819a089f33550631ed5b7c4847276496e869cbd6918d37a9176dd007ad100fd","pr_number":6770,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2dc905970642341b5133d06f9629d7da5a13553cc2ccde3c27e0d2e405210927","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6750650e460\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x6750650e400\u003e{\n            observation: \u003c*errors.errorString | 0x6750816aa40\u003e{\n                s: \"dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x6750650e3e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x67506232820\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x6750650e3c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x67507892480\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6750650e460\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wbzzkmqrv5j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x6750650e400\u003e{\n            observation: \u003c*errors.errorString | 0x6750816aa40\u003e{\n                s: \"dev/westus3/ci01-j9298304-mgmt-2/ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x6750650e3e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x67506232820\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x6750650e3c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j9298304-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tnnvmiqd0ho3pieph8vtukoridbmhhh-t4k1z6w0y4s5q4h\\\"; let start = datetime(2026-10-28T10:25:27.2654774Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x67507892480\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376655219298304","occurred_at":"2026-10-28T09:34:31Z","signature_id":"da049643627fdd3d8b49937e7c216abea494324607b970c619b0669a8ec08194","pr_number":6718,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"818794cda20cc99c353a743beee80b93b7a1886f28dfb74144dded0f0d86e651","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2680bac6c1a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2680bac6c180\u003e{\n            observation: \u003c*errors.errorString | 0x2680b95b3860\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2680bac6c160\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2680b9c4c190\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2680bac6c140\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2680bb4eaab0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2680bac6c1a0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2680bac6c180\u003e{\n            observation: \u003c*errors.errorString | 0x2680b95b3860\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2680bac6c160\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2680b9c4c190\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2680bac6c140\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j1816448-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\\\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2680bb4eaab0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114918973651816448","occurred_at":"2026-10-27T03:15:51Z","signature_id":"afd200cebfca006fbbf28cf7ff8dfd7883685e35a9e278667284486e031f242f","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7cd226b1ef7b8222b38971a112303a8516ff1f0851d469499d37f592a43adc2a","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T21:36:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32b05a79ea40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x32b05c5389f0\u003e{\n            s: \"dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32b05a79ea40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x32b05c5389f0\u003e{\n            s: \"dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113384044728160256","occurred_at":"2026-10-22T21:36:35Z","signature_id":"d6bbcd1d10bc7d627f007c65311d55d2d53b06f7912e2f2c8d382d44b94fe9f5","pr_number":7667,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a346f2ee38025c72bdbab745d0e046d927c6651ff3fe7f95b5953a94c870fd22","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=\u003cuuid\u003e pass=\u003cuuid\u003e: expected healthy complete coverage","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T05:21:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n        err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n            s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n        err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n            s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"7dc8c3eafac5404affbb8c6a5aceaf23d8664d06a783b6429405a361c74eae60","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1591afee4f2df55f5110e8b81c5b4958d76dbcd81b95ec7776997e6bc1915980","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x910473e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x91047780d20\u003e{\n            s: \"dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x910473e2000\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x91047780d20\u003e{\n            s: \"dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7787/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115208311295971328","occurred_at":"2026-10-27T22:25:34Z","signature_id":"8cbd1f8525d7ee50ddb682a6839345188c008a0a6039e30cba962b84e2ffdd23","pr_number":7787,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"be0a16e09bfa0f5959b8c6c18e735d670abbd8db1c2e1fc90c731326c8bcaa62","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T00:55:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x305427581240\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x305428017c50\u003e{\n            s: \"dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x305427581240\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        err: \u003c*errors.errorString | 0x305428017c50\u003e{\n            s: \"dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113071812228157440","occurred_at":"2026-10-22T00:55:53Z","signature_id":"4f129739bafddc5ac315cccd2d58e2f04f54ce8381b65dab5a26632d8e3d25a0","pr_number":7592,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d06537352cd3471c3b96668b882a7f97e44d69999f5ee5ae3c9e3cfa12840861","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22cffa244a80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x22cffa418620\u003e{\n            s: \"dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22cffa244a80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x22cffa418620\u003e{\n            s: \"dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114700407203696640","occurred_at":"2026-10-26T12:47:20Z","signature_id":"02547a32e2a28fef0dfb7aef6df6795a8819c05a7a7e03db3b64156256e4e2aa","pr_number":7757,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"eec089ccbb8d0a7f750b775b5e342570b85e76ed163473d22158ec1f98398c36","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b13ff3f4aa0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2b13ffa1a410\u003e{\n            s: \"dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b13ff3f4aa0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        err: \u003c*errors.errorString | 0x2b13ffa1a410\u003e{\n            s: \"dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7704/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115364484842262528","occurred_at":"2026-10-28T08:46:09Z","signature_id":"78dd0668e8380bd8ed00795b6ff7617e72c8fc53789f3998be7247874f824a7b","pr_number":7704,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4c7f20d280537e26756f2d14af08d1d2b02d836200b57c788ddc44687fa7d058","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2555ba2ee820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x2555ba2ee800\u003e{\n            observation: \u003c*errors.errorString | 0x2555b9ce7460\u003e{\n                s: \"dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2555ba2ee7e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2555b95a8370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2555ba2ee7c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2555baea86f0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2555ba2ee820\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*verifiers.pollObservationError | 0x2555ba2ee800\u003e{\n            observation: \u003c*errors.errorString | 0x2555b9ce7460\u003e{\n                s: \"dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2555ba2ee7e0\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2555b95a8370\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2555ba2ee7c0\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"westus3\\\"; let cl = \\\"ci01-j7087616-mgmt-2\\\"; let ns = \\\"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\\\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2555baea86f0\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114982798447087616","occurred_at":"2026-10-27T07:29:28Z","signature_id":"7e879e25f61ce940e144bedff05c8b377c887bfe979c26ff189023f3cf18a17c","pr_number":7598,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a8095a8b6eafff92d8870acf81e6750b45dca434bf91a8acb825aeaecf138193","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)","search_query":"[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-g455dk\"\nUnexpected error:\n    \u003c*errors.joinError | 0x366719114510\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\n    [pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-7vqg4/e2e-csi-pvc\" (reason=Provisioning, count=10, age=44s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x366718da0b80\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                err: \u003c*fmt.wrapError | 0x366718677020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x366718da09e0\u003e{\n                        observation: \u003c*errors.errorString | 0x366717e20d40\u003e{\n                            s: \"pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x366718da09c0\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-7vqg4/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-g455dk\"\nUnexpected error:\n    \u003c*errors.joinError | 0x366719114510\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \"Pending\", expected Succeeded; PVC is Bound\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\n    [pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodReadyToStartContainers status=False reason= message=\n    [pod] condition type=Initialized status=True reason= message=\n    [pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\n    [pod] condition type=PodScheduled status=True reason= message=\n    [pod] container volume-check ready=false restartCount=0\n    [pod]   waiting reason=ContainerCreating message=\n    [events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-7vqg4/e2e-csi-pvc\" (reason=Provisioning, count=10, age=44s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\n    [events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x366718da0b80\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                err: \u003c*fmt.wrapError | 0x366718677020\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.003s: pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-7vqg4/e2e-csi-pvc phase=Bound volumeName=pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 storageClass=managed-csi\\n[pod] e2e-csi-disk-7vqg4/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodReadyToStartContainers status=False reason= message=\\n[pod] condition type=Initialized status=True reason= message=\\n[pod] condition type=Ready status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=ContainersReady status=False reason=ContainersNotReady message=containers with unready status: [volume-check]\\n[pod] condition type=PodScheduled status=True reason= message=\\n[pod] container volume-check ready=false restartCount=0\\n[pod]   waiting reason=ContainerCreating message=\\n[events] e2e-csi-disk-7vqg4 Pod/e2e-csi-pod: Successfully assigned e2e-csi-disk-7vqg4/e2e-csi-pod to w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=Scheduled, count=0, age=43s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-7vqg4/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=44s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node w0w6l5s0n2h3v2v-np-g455dk-ggxrg-wrhrv (reason=ProvisioningFailed, count=8, age=7m38s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m45s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=1, age=5m0s)\\n[events] e2e-csi-disk-7vqg4 PersistentVolumeClaim/e2e-csi-pvc: Successfully provisioned volume pvc-4df3e109-ab4c-40f2-a58f-f8ec1ed79458 (reason=ProvisioningSucceeded, count=1, age=43s)\",\n                    err: \u003c*verifiers.pollObservationError | 0x366718da09e0\u003e{\n                        observation: \u003c*errors.errorString | 0x366717e20d40\u003e{\n                            s: \"pod e2e-csi-disk-7vqg4/e2e-csi-pod phase is \\\"Pending\\\", expected Succeeded; PVC is Bound\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0x366718da09c0\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-7vqg4/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7755/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115061887891673088","occurred_at":"2026-10-27T12:43:44Z","signature_id":"54d56dfc4a800362ca3c14d435d3f0db9206680219914967c341326c8f6b19e0","pr_number":7755,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"266fecc1d3355c9965ab1cf579cbe09c8cc3762b85e6438d6ae6f2889f5af561","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)","search_query":"[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dj4ptk\"\nUnexpected error:\n    \u003c*errors.joinError | 0xd5290aa85b8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xd5290a963a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                err: \u003c*fmt.wrapError | 0xd5290356ce0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                    err: \u003c*verifiers.pollObservationError | 0xd5290356540\u003e{\n                        observation: \u003c*errors.errorString | 0xd52904125d0\u003e{\n                            s: \"PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0xd5290356520\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dj4ptk\"\nUnexpected error:\n    \u003c*errors.joinError | 0xd5290aa85b8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xd5290a963a0\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                err: \u003c*fmt.wrapError | 0xd5290356ce0\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-8ql64/e2e-csi-pvc\\\" (reason=Provisioning, count=11, age=10s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)\",\n                    err: \u003c*verifiers.pollObservationError | 0xd5290356540\u003e{\n                        observation: \u003c*errors.errorString | 0xd52904125d0\u003e{\n                            s: \"PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                        },\n                        interruption: \u003c*fmt.wrapError | 0xd5290356520\u003e{\n                            msg: \"failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded\",\n                            err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115174942373842944","occurred_at":"2026-10-27T20:12:58Z","signature_id":"ac88b44d49d4b3fc31eb516f5c18cc6ed9c7017beeca05d29331dc6a4f6b0be6","pr_number":6081,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba04fee393b18d4d6b04ac1f68b72a6345177ebad95a48fd67494f2e5f295e14","failure_pattern":"[events] e2e-csi-disk-\u003cid\u003e PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)","search_query":"[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-6lzc2b\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263fb059e4c8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x263fb0784660\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                err: \u003c*fmt.wrapError | 0x263fb09fc440\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                    err: \u003c*errors.errorString | 0x263fb0ff0b50\u003e{\n                        s: \"PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-6lzc2b\"\nUnexpected error:\n    \u003c*errors.joinError | 0x263fb059e4c8\u003e: \n    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n    [storageclass] name=managed-csi provisioner=disk.csi.azure.com\n    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x263fb0784660\u003e{\n                msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                err: \u003c*fmt.wrapError | 0x263fb09fc440\u003e{\n                    msg: \"VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded\\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \\\"VolumeBinding\\\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \\\"e2e-csi-disk-sfxr9/e2e-csi-pvc\\\" (reason=Provisioning, count=10, age=2m54s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)\",\n                    err: \u003c*errors.errorString | 0x263fb0ff0b50\u003e{\n                        s: \"PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \\\"Pending\\\", expected Bound; pod phase is \\\"Pending\\\"\",\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127036694499328","occurred_at":"2026-10-27T17:02:37Z","signature_id":"d64f17ee0348ee472ddc21b645d0031133903222e72b9d637406698ac14c2738","pr_number":6081,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-20T18:05:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-28T13:22:31Z\n  Ended:       2026-10-28T13:28:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6945280-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.108:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/capi-provider is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115409675166945280","occurred_at":"2026-10-28T11:45:41Z","signature_id":"497a407fa6899d41490c9159b892ae9c0b2b92661b43d243f6d927eeba7fd4b9","pr_number":7782,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j4712832 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-22T20:58:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j4712832 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-22T22:34:51Z\n  Ended:       2026-10-22T22:41:02Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113374411854712832","occurred_at":"2026-10-22T20:58:18Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j4712832 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-22T20:58:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j4712832 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-22T22:23:37Z\n  Ended:       2026-10-22T22:51:36Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113374411854712832","occurred_at":"2026-10-22T20:58:18Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-22T20:25:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T21:30:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j3800704-mgmt-1 has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","signature_id":"f42018e30e12f894f1629fdf1b2b27fda4bc7322b3ae6f164b98305381f5a376","pr_number":7703,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6650ca1290fb5ed06584e1395a3de82b930b52fc740efa2bb1408633ebdaa7dd","failure_pattern":"alert [svc] FluentBitIngestionPaused fired","search_query":"alert [svc] FluentBitIngestionPaused fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FluentBitIngestionPaused does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:54:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FluentBitIngestionPaused does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-25T11:48:54Z\n  Ended:       2026-10-25T11:58:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"FluentBitIngestionPaused\", cluster=\"ci01-j7524864-mgmt-2\", component=\"monitoring-infrastructure\", pod=\"arobit-forwarder-q4jd5\", region=\"westus3\", severity=\"warning\"\n  Description: Fluent Bit pod arobit-forwarder-q4jd5 on cluster ci01-j7524864-mgmt-2 has paused collecting new log data for at least 5 minutes.\nIngestion pauses when Fluent Bit\\'s internal memory or storage buffers are full, typically caused by backpressure from a slow or failing output.\nInvestigate the Fluent Bit logs for the specific error details and check the Kusto instance health."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294563647524864","occurred_at":"2026-10-25T09:54:40Z","signature_id":"98720abe90ff53bf328a9b4ffc00853a75870d3a5d84137af7e1ce00be424528","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-16T14:51:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:35:36Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j8270720-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.66:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8270720/providers/microsoft.monitor/accounts/services-j8270720\", microsoft.metrics_include_label=\"service\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-5c8c49b687-k7qwk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-28T00:35:36Z\n  Severity:    Sev2\n  Labels:      alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j8270720-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.66:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8270720/providers/microsoft.monitor/accounts/services-j8270720\", microsoft.metrics_include_label=\"service\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-5c8c49b687-k7qwk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","signature_id":"77f4969f1613eac390c074f1db2e579bea3998e6e11e045a055ac7f824623d4f","pr_number":5244,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fb0cbad5198bdae4515e07599d2b1b6ff3ac6cb7da7109069ce8cd804ef1fb4b","failure_pattern":"alert [svc] PrometheusJobUp fired","search_query":"alert [svc] PrometheusJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T20:25:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T21:19:53Z\n  Severity:    Sev2\n  Labels:      alertname=\"PrometheusJobUp\", cluster=\"ci01-j3800704-mgmt-1\", component=\"monitoring-infrastructure\", severity=\"critical\"\n  Description: Prometheus has not been reachable for the past 10 minutes.\nThis may indicate that the Prometheus server is down, unreachable due to network issues, or experiencing a crash loop.\nCheck the status of the Prometheus pods, service endpoints, and network connectivity."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7703/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113366191513800704","occurred_at":"2026-10-22T20:25:38Z","signature_id":"8622c4f05e530429c2435ac6bd159aff27f3109548086ec61b7cd9ecf8f87649","pr_number":7703,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ed62f5f6e15089a288745d0cb27e1873c68853958b7d80abb95ff11fd25b0be8","failure_pattern":"cert-manager rollout in progress: updated=1 ready=1 desired=2","search_query":"cert-manager rollout in progress: updated=1 ready=1 desired=2","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T02:56:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\ncert-manager deployment should complete rollout with workload identity\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x224df5000400\u003e: \n    cert-manager rollout in progress: updated=1 ready=1 desired=2\n    {\n        s: \"cert-manager rollout in progress: updated=1 ready=1 desired=2\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\ncert-manager deployment should complete rollout with workload identity\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x224df5000400\u003e: \n    cert-manager rollout in progress: updated=1 ready=1 desired=2\n    {\n        s: \"cert-manager rollout in progress: updated=1 ready=1 desired=2\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113827015499976704","occurred_at":"2026-10-24T02:56:48Z","signature_id":"8c262f00ec05d49570da12d0f71a5cb18b3aa7c5602d40ca3360b5c689bea7bb","pr_number":7514,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":4,"also_in":["STG"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-25T12:30:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x25b4e2d279a0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x25b4e2d279a0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333668922101760","occurred_at":"2026-10-25T12:30:03Z","signature_id":"5ceb3b0e570a6fbb8e99110224b2256342313e80153bb81dc65b77bbad401dd6","pr_number":6191,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-19T11:29:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x3462d072440?, {0x7f705437c4e0?, 0x3462db47650?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1197 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x3462df2c480, {0x7f705437c4e0, 0x3462db47650}, {0xfd7ebbf, 0xa}, {0x3462cfcc129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func27.2.1({0x7f705437c4e0, 0x3462db47650})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:141 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x105c3910?, 0x3462db47650?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 80\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x3462d072440?, {0x7f705437c4e0?, 0x3462db47650?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1197 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x3462df2c480, {0x7f705437c4e0, 0x3462db47650}, {0xfd7ebbf, 0xa}, {0x3462cfcc129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func27.2.1({0x7f705437c4e0, 0x3462db47650})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:141 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x105c3910?, 0x3462db47650?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 80\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:537]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114922665838907392","occurred_at":"2026-10-27T03:30:25Z","signature_id":"9695fdc7ed44a5281f86995428490bd94b9fe5e26b6dc18c50bb5151a4ed31bb","pr_number":7767,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a801ead30ba9f1a519d4f6e8321847a7265bbd197dc70d8fa10568c3510a622a","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5601792-mgmt-1\",\"clusterName\":\"ci01-j560179","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-27T15:30:56.681953143Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.731109745Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.873423526Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-27T15:30:56.670Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-27T16:00:56.129Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-27T15:30:56.681953143Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.731109745Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-27T15:30:58.873423526Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5601792-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5601792-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115093743005601792","occurred_at":"2026-10-27T14:50:19Z","signature_id":"8aa20315f26c0b32e08e3b819a65f71c1dceb69f32d20ac323dffb320bb45d77","pr_number":7670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"04bb1c86673a7de744e4cf404843226187998893a7858cc6ec66a84dbfdf8c20","failure_pattern":"failed to create service principal: create service principal for appId \"\u003cuuid\u003e\": Post \"\u003curl\u003e\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)","search_query":"failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b303bad2a00\u003e: \n    failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\n    {\n        msg: \"failed to create service principal: create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n        err: \u003c*fmt.wrapError | 0x1b303bad2980\u003e{\n            msg: \"create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n            err: \u003c*url.Error | 0x1b303c6b42d0\u003e{\n                Op: \"Post\",\n                URL: \"https://graph.microsoft.com/v1.0/servicePrincipals\",\n                Err: \u003c*http.timeoutError | 0x1b303d466f50\u003e{\n                    err: \"net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b303bad2a00\u003e: \n    failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\n    {\n        msg: \"failed to create service principal: create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n        err: \u003c*fmt.wrapError | 0x1b303bad2980\u003e{\n            msg: \"create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n            err: \u003c*url.Error | 0x1b303c6b42d0\u003e{\n                Op: \"Post\",\n                URL: \"https://graph.microsoft.com/v1.0/servicePrincipals\",\n                Err: \u003c*http.timeoutError | 0x1b303d466f50\u003e{\n                    err: \"net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114665780556599296","occurred_at":"2026-10-26T10:29:42Z","signature_id":"4f64685c87a06c2999566f5fae5dd314d3059900665f264709b0778579c1b337","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-14T15:52:39Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-27T17:52:24.476Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-27T17:52:24.848Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-27T17:52:24.848Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T17:52:24.880Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 namespace=multicluster-engine\ntime=2026-10-27T17:52:24.933Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 dryRun=false\ntime=2026-10-27T17:52:24.933Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T17:52:24.990Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2\ntime=2026-10-27T18:02:49.740Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-27T18:02:49.741Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7753/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115127534017318912","occurred_at":"2026-10-27T17:04:34Z","signature_id":"4fbcbcae7d0f352f867128831decf8a2cd0c0541228c3392fa864aa36c4d6550","pr_number":7753,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9a8320dfc4cb472661bbb97b2880f78576cae510dd7ae52938db3db0330d5778","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-28T17:01:13.002Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-28T17:01:13.550Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-25 azureArmHelperIdentityClientId:ea5e9c6c-1933-464d-8bf6-db343c22b92d azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:ae6cf536-acc0-414c-ada8-f268aeeb97e3 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j3195648 cosmosDatabaseUrl:https://arohcpci01-rp-j3195648.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495] imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi monitoringApiGroup:monitoring.coreos.com ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj3195648.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj3195648.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:a5d5ff83-799c-44fd-a535-8c4386588d02 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-28T17:01:13.550Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:01:13.717Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-28T17:01:13.899Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-28T17:01:13.899Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:01:13.952Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-28T17:11:19.322Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded\"\ntime=2026-10-28T17:11:19.322Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115478705303195648","occurred_at":"2026-10-28T16:20:01Z","signature_id":"b40184a1cf5ce039c231fc664b3fad303d9d873ec2b7da0a9a97597e803d23e4","pr_number":7311,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c9b30ffa48a46e2fe98d38066e691b7b2af650635955d57fc11c75ad9e01b688","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-28T00:24:38.589Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-28T00:24:39.214Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:80c01b85-e07b-47ba-b2c6-d25569ff69d4 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j8270720.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j8270720] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:aba8ed3c2d6e8b0f6a5e17b2f65dba996c5cf673b73a35cc550521873b08c231 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] monitoringApiGroup:monitoring.coreos.com pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:d99261bb-4b6f-46d6-96d4-bae9c86ef284 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-10-28T00:24:39.214Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:24:39.557Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-10-28T00:24:39.898Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-10-28T00:24:39.898Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:24:40.025Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-28T00:34:46.993Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"\ntime=2026-10-28T00:34:46.994Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115222887018270720","occurred_at":"2026-10-27T23:23:29Z","signature_id":"ef8a2bb1c5a90008d03e66f07016c90f57a640a5c8bdad13633717d011c3e6b7","pr_number":5244,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T00:46:11Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1}],"full_error_samples":["time=2026-10-22T01:22:39.748Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-10-22T01:22:40.100Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:f45f2e02544ba4005d39e435073a6c59ab5db372223dee379b08a36d3bc20871 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-10-22T01:22:40.101Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:22:40.245Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring\ntime=2026-10-22T01:22:40.391Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false\ntime=2026-10-22T01:22:40.391Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:22:40.452Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-22T01:27:43.190Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-22T01:27:43.190Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113069370304368640","occurred_at":"2026-10-22T00:46:11Z","signature_id":"6b8d9984b55592851b0841c5e8c4a0f18f8f61520e7df4a51c7e6afc9a707725","pr_number":7674,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-22T12:06:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1971c8790ec0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-qg2q7d744n2t\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1971c8790ec0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-qg2q7d744n2t\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113240554585198592","occurred_at":"2026-10-22T12:06:24Z","signature_id":"0c413ba7400d39619499c18f717f654ad4d24d7d3d25c4fd556162f570f4d3c4","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5561db6163db8e6282579645be22b666c3caa655cd5984b6509f9f92368e2bd6","failure_pattern":"image pull not confirmed; cause undetermined (insufficient evidence)","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T16:08:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ac7e2dd140\u003e: \n    image pull not confirmed; cause undetermined (insufficient evidence)\n    polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"image pull not confirmed; cause undetermined (insufficient evidence)\\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*fmt.wrapError | 0x35ac7e2dd120\u003e{\n            msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n            err: \u003c*errors.errorString | 0x35ac7fce7bf0\u003e{\n                s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ac7e2dd140\u003e: \n    image pull not confirmed; cause undetermined (insufficient evidence)\n    polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"image pull not confirmed; cause undetermined (insufficient evidence)\\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*fmt.wrapError | 0x35ac7e2dd120\u003e{\n            msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n            err: \u003c*errors.errorString | 0x35ac7fce7bf0\u003e{\n                s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113301360286896128","occurred_at":"2026-10-22T16:08:02Z","signature_id":"3e9aaa4d1d813a4709137ed455a6a028fd10e68d2b9d24f9a1cbc8601d7c3757","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e20314db8c9a8fa7947e904463dad9423d8152bc1bcd1896b51f5df0a9fd524f","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T13:32:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0x390d19dbe00\u003e: \n    list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\n    {\n        msg: \"list nodes (nodePool=npupgrade-4-20): Get \\\"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\\\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n        err: \u003c*url.Error | 0x390d1c076e0\u003e{\n            Op: \"Get\",\n            URL: \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\",\n            Err: \u003c*net.OpError | 0x390d1655400\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x390d16552c0\u003e{\n                    UnwrapErr: nil,\n                    Err: \"read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n                    Name: \"api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud\",\n                    Server: \"172.30.0.10:53\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0x390d19dbe00\u003e: \n    list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\n    {\n        msg: \"list nodes (nodePool=npupgrade-4-20): Get \\\"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\\\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n        err: \u003c*url.Error | 0x390d1c076e0\u003e{\n            Op: \"Get\",\n            URL: \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\",\n            Err: \u003c*net.OpError | 0x390d1655400\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: nil,\n                Err: \u003c*net.DNSError | 0x390d16552c0\u003e{\n                    UnwrapErr: nil,\n                    Err: \"read udp 172.24.233.194:52762-\u003e172.30.0.10:53: i/o timeout\",\n                    Name: \"api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud\",\n                    Server: \"172.30.0.10:53\",\n                    IsTimeout: true,\n                    IsTemporary: true,\n                    IsNotFound: false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113262158069370880","occurred_at":"2026-10-22T13:32:15Z","signature_id":"c38efd6ad70105db427bd0e6d9a57dde87e6596812ce842dbcdfefe00ee77962","pr_number":7653,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7cf505551991c2dd368b5369e4fad74b682af51f37d72a779eab2ae641b8364d","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T09:53:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2f3433d553d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2f3433d553d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10782, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114294348290985984","occurred_at":"2026-10-25T09:53:48Z","signature_id":"8d9d7ef8eccc46082983c78c6940e3a331458157cfab732b80148db78e3f5fd5","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87b6fa8784f23f8d704a5e8b3db46de48e9b95d9d70ba77988130bb26adb90a4","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x283ddd596200\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x283ddd596200\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115025503944445952","occurred_at":"2026-10-27T10:19:07Z","signature_id":"ace47547bb37bbb48ad9829ba0c4939a4085b942cd0b112d65b243d63eb8c4f2","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"20d58027fbe6fa8b683210764894d620c01183ffc035b6c390592701e4ed6fda","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T04:30:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 601.097s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x11218fd9090\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 601.097s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x11218fd9090\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850692501245952","occurred_at":"2026-10-24T04:30:53Z","signature_id":"2437bca1dfdab8003e0f75fb95e845e5a73877ed5b1c32672694464dd4e2adda","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d516b1df04e827a663b514ce5e826565130872a9cbbbc30e6eebf3f39c4bbbab","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xca1fe06c6d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xca1fe06c6d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115416945330229248","occurred_at":"2026-10-28T12:14:35Z","signature_id":"43128ad58b9794b202ee869cfaa57955ea19870ba55aba4a0092b4e1c16edd9c","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"471610528d575f0ca7dcf6b5555ac33936f790050d7e1c045a353856d7e30af3","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe6261437860\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe6261437860\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114613195741073408","occurred_at":"2026-10-26T07:00:36Z","signature_id":"51acba6d51b844246e9a06873b1fe5c6da4f4a9182dfef26ae27d3095e26d5a8","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"25bd34308f5a077a278630c24a7efd62cb72a50de20cc84690294e8ce77710e0","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3952e9095e40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3952e9095e40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","signature_id":"56a5235eada3f8c8a737b876653b1f4d48462800a44eefb3574680f288886ed9","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"73bb7634af35e831eaf0b0bd52966c2235200ca7b8f76813dbe2e28963b542ce","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T12:30:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 620.639s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1cfcad623430\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 620.639s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1cfcad623430\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14128, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114333714598072320","occurred_at":"2026-10-25T12:30:14Z","signature_id":"090ccca170527876a3cb6a1108982d860d57ba3a86683c697ef9e7b7b0f84311","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40550f1869e8f917cf0ba372be4f16663c844fbaf77e07168049278696500fd1","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T17:30:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 630.763s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2d6e687aea00\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 630.763s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2d6e687aea00\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14240, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114409269653671936","occurred_at":"2026-10-25T17:30:28Z","signature_id":"4089890b6b331518bf34a303952645ae56025b4bf66de4696178dbcd0ea811a5","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3cb444de9b5329e96be0f474504d91a498322635b3598692d70b1c8025a75f8b","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x867032a97d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x867032a97d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115285263486816256","occurred_at":"2026-10-28T03:31:21Z","signature_id":"6ce95bc6ecd9c4fb5878916c5f1f43531cb7d0bd302c877f171028ac537ce75e","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3921dee80f90c25a3f2a89623ede0bae1ddb59e58283210f34e1352a8c2ad0b2","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x20bd1e04e7a0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x20bd1e04e7a0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115339656903004160","occurred_at":"2026-10-28T07:07:29Z","signature_id":"2ed854a334795c3ea51fafdf41aa1c84cc3b8fdf14b1788a9d3550323b78a2db","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4ce5bd36c087ccb995c35f95947eba01b62e03f0c2baf17243f6a3723c9f1559","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de4f1b0a5d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de4f1b0a5d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2115376885591445504","occurred_at":"2026-10-28T09:35:25Z","signature_id":"5ae5d9bea9a6a2cf4aba7447363670b8392610f5289f3a39616120110fd0bfc0","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d5faba6a3467bcc4356e02ff8b763d0cb27996ad0cf0c261ffce5c17c56051d3","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x12a5a347cf40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x12a5a347cf40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114650067662016512","occurred_at":"2026-10-26T09:27:19Z","signature_id":"ba0c38e1eca9a292d0e25be503d8588cc9faffce90f396b1e444f3df42590094","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4d661fb661180412d79fc99146fbf8d02151376731f8cbdb584bccc883aaf3e0","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x211f21252720\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x211f21252720\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114974410921218048","occurred_at":"2026-10-27T06:56:08Z","signature_id":"f56a1a7fe0529dc820427621b1f6eaead9f3912f807ee59dc2e3a11a74974b94","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e6a6ef637ffd01a8f5540827ca11eebe7cd606076b9508c9bc2e31657ba24b47","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x294c3a4e6f50\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x294c3a4e6f50\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","signature_id":"153f0aaddc00f1ad7874f65ac4a270a70a9ea96971f5237b887bc92ff2778128","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24f84e8da288b18c5a6ce017a06862a7753a25291f7369a28e80acece5813bd2","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T18:38:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"97dc8f8152c396287fe01fe00d78417ce23034d6cc4cad15e28215178e3bc6eb","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dd8293942caca74f6fdf5f2f77a272ddca84daf5e523e4bbdebdc477d7c3ee71","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T18:32:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3863a797e030\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3863a797e030\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113337766480711680","occurred_at":"2026-10-22T18:32:42Z","signature_id":"4249d51f56b51a2a4f36f216d451e5000bdfdbe8cc19e478c64f0ed465dd4943","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b727ea5a9ba2a3858b4292ab83d8c90a401379a8446650e11bad43c6ae5ed290","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T00:51:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de7945ac470\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de7945ac470\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795606852931584","occurred_at":"2026-10-24T00:51:59Z","signature_id":"2478f6179eaff1c358edd52ab231bf86c77990919d3c9e804395e4e4562a42aa","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"729ad653b1a4e1df171b5384e8c8a7d6ecb54a27a28c07489437e4661c195409","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T11:13:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3fcfb02035d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3fcfb02035d0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113227265989939200","occurred_at":"2026-10-22T11:13:36Z","signature_id":"8de204c51ea1e77126cc08101aced92a4c4f1946ec3be586cda6f8a7ac4a26df","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3aa90bdf47475703df3b110c727dbfb72dbdbd02636c9ba15471acd53cf0021f","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T07:04:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x291503b87830\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x291503b87830\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113164536595877888","occurred_at":"2026-10-22T07:04:20Z","signature_id":"3b8862f53308a60b2af78a86bb8911332deef646ed130d7021031cd841820f0b","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c1c8b39ba5d3ca3c70c30864a182ac9ff6f4361297590b1691f87bbb39aadd6d","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T15:20:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xf9bfe384ae0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xf9bfe384ae0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7690/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113289300694011904","occurred_at":"2026-10-22T15:20:06Z","signature_id":"904e103fc4d240f7d05c2e2efb5e56be203c4cc87483a19c18cf88198cb1b1d6","pr_number":7690,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T05:21:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n    \u003cint\u003e: 0\nto be \u003e=\n    \u003cint\u003e: 2\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n    \u003cint\u003e: 0\nto be \u003e=\n    \u003cint\u003e: 2"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"93484e969eefe4c20a3e175e375caad69b1f31f45ea3188e5adb186dca3ed1fc","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","failure_pattern":"pod targeting cordoned node \u003cnode\u003e should not be assigned a node","search_query":"pod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T10:30:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.\npod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node\nExpected\n    \u003cstring\u003e: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.\npod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node\nExpected\n    \u003cstring\u003e: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113216309620641792","occurred_at":"2026-10-22T10:30:03Z","signature_id":"535db7791df0e62f6c566e8206080df859bf280615373999df4bcace6f1dde18","pr_number":7598,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36030c2f6530fdbd07d634405c15916c6fea00f3911df6853eed3a48b722f69c","failure_pattern":"timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"\u003cvm\u003e\"; canceled run command","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T07:08:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0x339af03e78d0\u003e: \n    timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command\n    {\n        s: \"timed out waiting for VM run command \\\"e2e-runcommand-1792657320988467170\\\" on VM \\\"cidr-connectivity-test-test-vm\\\"; canceled run command\",\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0x339af03e78d0\u003e: \n    timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command\n    {\n        s: \"timed out waiting for VM run command \\\"e2e-runcommand-1792657320988467170\\\" on VM \\\"cidr-connectivity-test-test-vm\\\"; canceled run command\",\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113165525352714240","occurred_at":"2026-10-22T07:08:16Z","signature_id":"bb30606bde067b70e417b1ab7034bb648603ed8a2a28fe33e585197379c8df85","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.21141649048625794,"anchor_occurrences":1,"after_last_push_count":23,"trend_counts":[9,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":71,"prior_last_seen_at":"2026-10-22T12:49:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bbfb1338400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1bbfb13383e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x1bbfb092c1e0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x1bbfb13383c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x1bbfb280d260\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bbfb1338400\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1bbfb13383e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x1bbfb092c1e0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x1bbfb13383c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x1bbfb280d260\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113251417044029440","occurred_at":"2026-10-22T12:49:34Z","signature_id":"1925c4ff43c7c8127f497ba14b75cd670537fb5f70933201f320fd96cbf2c0c3","pr_number":7547,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":43,"failed_runs":28,"raw_occurrences":50,"matched_occurrences":46,"runs_affected":24},"rows":[{"environment":"int","failure_pattern_id":"bb891a93a65cf93fcde8354314a61c27dbb024eae76346ca347cf3e3744fffe5","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":27.906976744186046,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,5,7,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-24T22:15:18Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":12}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-l96rfk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11cfecaf8760\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x11cfee5e70e0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x11cfee5e70b0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x11cfee823170\u003e{\n                            N: {\n                                neg: false,\n                                abs: [12328844732508617451, 3386053924852305530, 11508787352079574519, 580165219016783073, 15126087406076015328, 8473747687808762055, 6638534397921598209, 4531117369475629035, 8686734636532653775, 9699186360867822250, 15005114980278798855, 3900631334283717631, 15336487295176330797, 13597435546774256175, 1139063161098112232, 10161195232013165903, 15551291529941624294, 8053304939740217029, 18299432943387333464, 12035498982612883405, 1076299365375342957, 3325077180629581387, 4778720590180299818, 5926909629444315306, 16602979226450151451, 14838171221845856625, 18308053321550668292, 3951715190352458341, 11733219666321171503, 11793477282006375930, 17943060274730047763, 12841897204482256645],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [8845870848929380208],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928477681, loc: nil},\n                        NotAfter: {wall: 0, ext: 63960013681, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],\n                        AuthorityKeyId: [248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x11cfed040008 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-l96rfk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11cfecaf8760\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x11cfee5e70e0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x11cfee5e70b0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x11cfee823170\u003e{\n                            N: {\n                                neg: false,\n                                abs: [12328844732508617451, 3386053924852305530, 11508787352079574519, 580165219016783073, 15126087406076015328, 8473747687808762055, 6638534397921598209, 4531117369475629035, 8686734636532653775, 9699186360867822250, 15005114980278798855, 3900631334283717631, 15336487295176330797, 13597435546774256175, 1139063161098112232, 10161195232013165903, 15551291529941624294, 8053304939740217029, 18299432943387333464, 12035498982612883405, 1076299365375342957, 3325077180629581387, 4778720590180299818, 5926909629444315306, 16602979226450151451, 14838171221845856625, 18308053321550668292, 3951715190352458341, 11733219666321171503, 11793477282006375930, 17943060274730047763, 12841897204482256645],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [8845870848929380208],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928477681, loc: nil},\n                        NotAfter: {wall: 0, ext: 63960013681, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],\n                        AuthorityKeyId: [248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x11cfed040008 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-2rcw86\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b2e578984a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x2b2e57370780\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x2b2e57370750\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x2b2e57843d70\u003e{\n                            N: {\n                                neg: false,\n                                abs: [2877128800215149205, 16629919685921545055, 13732506504615110447, 11103748926260740575, 11416989105297106818, 9223988340828442177, 2897202458616478060, 11766261893188694937, 15086238765284949963, 10809216967110507236, 9979912210168109780, 11664503868611822661, 3024932991847888604, 17852206095763650445, 1698336212534998063, 17691123283498957122, 9751057521779199143, 5748498642685419890, 4821085446021960184, 2270194980282856385, 5998191060056422898, 1979344939107882427, 11002240530182040303, 17961926253159653156, 4444200366252054316, 17136494472856312874, 5046575580329891885, 18333653388545728381, 18332734810168546880, 2830450700003086872, 11572718067588258363, 13260460152132755496],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [8959707475658302546],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928472888, loc: nil},\n                        NotAfter: {wall: 0, ext: 63960008888, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],\n                        AuthorityKeyId: [145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x2b2e55ba2008 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-2rcw86\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b2e578984a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x2b2e57370780\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x2b2e57370750\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x2b2e57843d70\u003e{\n                            N: {\n                                neg: false,\n                                abs: [2877128800215149205, 16629919685921545055, 13732506504615110447, 11103748926260740575, 11416989105297106818, 9223988340828442177, 2897202458616478060, 11766261893188694937, 15086238765284949963, 10809216967110507236, 9979912210168109780, 11664503868611822661, 3024932991847888604, 17852206095763650445, 1698336212534998063, 17691123283498957122, 9751057521779199143, 5748498642685419890, 4821085446021960184, 2270194980282856385, 5998191060056422898, 1979344939107882427, 11002240530182040303, 17961926253159653156, 4444200366252054316, 17136494472856312874, 5046575580329891885, 18333653388545728381, 18332734810168546880, 2830450700003086872, 11572718067588258363, 13260460152132755496],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [8959707475658302546],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928472888, loc: nil},\n                        NotAfter: {wall: 0, ext: 63960008888, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],\n                        AuthorityKeyId: [145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x2b2e55ba2008 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-wk4gjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30fb7a9c6d80\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x30fb7cb9b320\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x30fb7cb9b2f0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x30fb7a9895b0\u003e{\n                            N: {\n                                neg: false,\n                                abs: [3713275565900749005, 5533862720488119315, 886043176200789153, 17818264461995138607, 16265450768615453130, 5834472357489801107, 5862144673391444614, 628656117149906395, 13178964295004987606, 715618696217184398, 12066541168516629782, 6050057140458382287, 14400619811794133371, 6862502860246136883, 1422013248764323211, 8372522752467421288, 13138907997452357860, 17826693266249031145, 5098815638656127509, 5738821655630622400, 12438080729488309683, 6057798802862875801, 7503648299216627878, 13557164215630681915, 9649412887563066602, 17200267080417426931, 12116969928591911697, 17745395029646614848, 8889710084128733264, 16509317662742049607, 11507875977146301207, 12660817078547036528],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [7949786151822990365],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928459200, loc: nil},\n                        NotAfter: {wall: 0, ext: 63959995200, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],\n                        AuthorityKeyId: [107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x30fb7a8f2c08 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-wk4gjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30fb7a9c6d80\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"[strict TLS verification] route was never reachable: Get \\\"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*url.Error | 0x30fb7cb9b320\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n            Err: \u003c*tls.CertificateVerificationError | 0x30fb7cb9b2f0\u003e{\n                UnverifiedCertificates: [\n                    {\n                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],\n                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],\n                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1],\n                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],\n                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],\n                        Signature: [62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],\n                        SignatureAlgorithm: 4,\n                        PublicKeyAlgorithm: 1,\n                        PublicKey: \u003c*rsa.PublicKey | 0x30fb7a9895b0\u003e{\n                            N: {\n                                neg: false,\n                                abs: [3713275565900749005, 5533862720488119315, 886043176200789153, 17818264461995138607, 16265450768615453130, 5834472357489801107, 5862144673391444614, 628656117149906395, 13178964295004987606, 715618696217184398, 12066541168516629782, 6050057140458382287, 14400619811794133371, 6862502860246136883, 1422013248764323211, 8372522752467421288, 13138907997452357860, 17826693266249031145, 5098815638656127509, 5738821655630622400, 12438080729488309683, 6057798802862875801, 7503648299216627878, 13557164215630681915, 9649412887563066602, 17200267080417426931, 12116969928591911697, 17745395029646614848, 8889710084128733264, 16509317662742049607, 11507875977146301207, 12660817078547036528],\n                            },\n                            E: 65537,\n                        },\n                        Version: 3,\n                        SerialNumber: {\n                            neg: false,\n                            abs: [7949786151822990365],\n                        },\n                        Issuer: {\n                            Country: nil,\n                            Organization: nil,\n                            OrganizationalUnit: [\"openshift\"],\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"root-ca\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 11],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"root-ca\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        Subject: {\n                            Country: nil,\n                            Organization: [\"openshift\"],\n                            OrganizationalUnit: nil,\n                            Locality: nil,\n                            Province: nil,\n                            StreetAddress: nil,\n                            PostalCode: nil,\n                            SerialNumber: \"\",\n                            CommonName: \"openshift-ingress\",\n                            Names: [\n                                {\n                                    Type: [2, 5, 4, 10],\n                                    Value: \u003cstring\u003e\"openshift\",\n                                },\n                                {\n                                    Type: [2, 5, 4, 3],\n                                    Value: \u003cstring\u003e\"openshift-ingress\",\n                                },\n                            ],\n                            ExtraNames: nil,\n                        },\n                        NotBefore: {wall: 0, ext: 63928459200, loc: nil},\n                        NotAfter: {wall: 0, ext: 63959995200, loc: nil},\n                        KeyUsage: 5,\n                        Extensions: [\n                            {\n                                Id: [2, 5, 29, 15],\n                                Critical: true,\n                                Value: [3, 2, 5, 160],\n                            },\n                            {\n                                Id: [2, 5, 29, 37],\n                                Critical: false,\n                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            },\n                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                            {\n                                Id: [2, 5, 29, 14],\n                                Critical: false,\n                                Value: [4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],\n                            },\n                            {\n                                Id: [2, 5, 29, 35],\n                                Critical: false,\n                                Value: [48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],\n                            },\n                            {\n                                Id: [2, 5, 29, 17],\n                                Critical: false,\n                                Value: \"0C\\x82A*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n                            },\n                        ],\n                        ExtraExtensions: nil,\n                        UnhandledCriticalExtensions: nil,\n                        ExtKeyUsage: [2, 1],\n                        UnknownExtKeyUsage: nil,\n                        BasicConstraintsValid: true,\n                        IsCA: false,\n                        MaxPathLen: -1,\n                        MaxPathLenZero: false,\n                        SubjectKeyId: [184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],\n                        AuthorityKeyId: [107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],\n                        OCSPServer: nil,\n                        IssuingCertificateURL: nil,\n                        DNSNames: [\n                            \"*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\",\n                        ],\n                        EmailAddresses: nil,\n                        IPAddresses: nil,\n                        URIs: nil,\n                        PermittedDNSDomainsCritical: false,\n                        PermittedDNSDomains: nil,\n                        ExcludedDNSDomains: nil,\n                        PermittedIPRanges: nil,\n                        ExcludedIPRanges: nil,\n                        PermittedEmailAddresses: nil,\n                        ExcludedEmailAddresses: nil,\n                        PermittedURIDomains: nil,\n                        ExcludedURIDomains: nil,\n                        CRLDistributionPoints: nil,\n                        PolicyIdentifiers: nil,\n                        Policies: nil,\n                        InhibitAnyPolicy: 0,\n                        InhibitAnyPolicyZero: false,\n                        InhibitPolicyMapping: 0,\n                        InhibitPolicyMappingZero: false,\n                        RequireExplicitPolicy: 0,\n                        RequireExplicitPolicyZero: false,\n                        PolicyMappings: nil,\n                    },\n                ],\n                Err: \u003cx509.UnknownAuthorityError\u003e{\n                    Cert: 0x30fb7a8f2c08 (cyclic reference),\n                    hintErr: nil,\n                    hintCert: nil,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114118563634614272","occurred_at":"2026-10-24T22:15:18Z","signature_id":"61bf447572f11e30f06b09d1a83c5a876cc25a3a6e0864bb540027ad0bb86be1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114094949644374016","occurred_at":"2026-10-24T20:41:28Z","signature_id":"054c567bc67fd2737a5e62176c21e69b0f603f7fa108fa2c588fe120729da589","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114034199856943104","occurred_at":"2026-10-24T16:40:04Z","signature_id":"a4a8a077386a0955c802844ae25e4e88d9f59843b49a9835ea13932646123fe1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113896710567628800","occurred_at":"2026-10-24T07:33:43Z","signature_id":"39db81ef1ca71f41814cb771e570746f428543825d6b6b767ed5b824cfe1caf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113874312019578880","occurred_at":"2026-10-24T06:04:44Z","signature_id":"f1df3f2de98e298fdce35dc438280fd6e026bad314a016066922950ccb4e8817","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113828436664389632","occurred_at":"2026-10-24T03:02:26Z","signature_id":"f7537ffd0c730ef1265c9e1b5bf3210466470816a6de65b90d9518de1a3e4e9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113793743264944128","occurred_at":"2026-10-24T00:44:35Z","signature_id":"fb65d237453759415adff6a794931ffeed1fcb92a486261b679d22c6ac07b556","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113768688329953280","occurred_at":"2026-10-23T23:05:01Z","signature_id":"d85b6e978d2f237313cfefb10961c683ac04080043f05a5f36dca38dfe514321","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113708197607903232","occurred_at":"2026-10-23T19:04:39Z","signature_id":"916d5cdac13cdd228e4a42a07c8bb8fe7a138fca50215f920616e116e1de786f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113681865297629184","occurred_at":"2026-10-23T17:20:01Z","signature_id":"e00b877db5887a60dafcdcc0dafcb829a26f18b80a249f9e86ac05ba2979c2d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113608404747423744","occurred_at":"2026-10-23T12:28:07Z","signature_id":"67d450e66ecc2534257ac7489b4bec1640f94851f32f4aeb32e0e18e4898283d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113565541661675520","occurred_at":"2026-10-23T09:37:46Z","signature_id":"b0409a71a63908ee1b2f80e0d0605733e605d1fcb88627a04cfeb048bfb9efcb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0e9c16ade6d351c6ad76f00bf3537f55905505199679274bea1149b0e7a9d4d6","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-rhgbbsmxx2nr\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.976744186046512,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,2,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-23T12:28:07Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1977dbbad170\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-c7xfsrnzff2g\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-c7xfsrnzff2g\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1977dcaa2220\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1977dbbad170\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-c7xfsrnzff2g\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-c7xfsrnzff2g\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1977dcaa2220\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2cc075309380\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-rhgbbsmxx2nr\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-rhgbbsmxx2nr\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2cc074e74090\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2cc075309380\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-rhgbbsmxx2nr\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-rhgbbsmxx2nr\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2cc074e74090\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x32cbbfa08480\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-s8rx2m7kc8lm\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-s8rx2m7kc8lm\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x32cbbf682f10\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x32cbbfa08480\u003e: \n    failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-s8rx2m7kc8lm\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-s8rx2m7kc8lm\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x32cbbf682f10\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113608404747423744","occurred_at":"2026-10-23T12:28:07Z","signature_id":"969a30400ed2f71fb417f2923bd10cb56a980623c6bf81cc1010f62c2c2fb7b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113221801671659520","occurred_at":"2026-10-22T10:51:53Z","signature_id":"cd88ae455d6681e4c62aa1fe194740b8f3c85d8c37000314a86501ed4ccbe1fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113188336309899264","occurred_at":"2026-10-22T08:38:55Z","signature_id":"c85b7a1a73c33a6f5d4a229bc06fe02d0e5cf761eb22acb2b1856681061514f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a3c2b9956e94f5bf89728ed173d00d94f6cd2937913ff1f5fca5ff2b598b1b1a","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.976744186046512,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,1,1,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-24T16:40:04Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xdad4da81e80\u003e: \n    DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xdad4d9b41a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xdad4da81e80\u003e: \n    DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xdad4d9b41a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-5f9zxt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3db0320ca900\u003e: \n    DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3db031a2bd20\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-5f9zxt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3db0320ca900\u003e: \n    DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3db031a2bd20\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1342e25441a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1342e251c060\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1342e25441a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1342e251c060\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114034199856943104","occurred_at":"2026-10-24T16:40:04Z","signature_id":"234b3dcd6b1b99d0fc0f94c1b7b07a5201359f043ca21f57a2bd1b8d5e4c4796","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113681865297629184","occurred_at":"2026-10-23T17:20:01Z","signature_id":"1e3ee217e6bd3fae46804afa3144ec156a85b863dc42482fc37485ea8190a8e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113114644393496576","occurred_at":"2026-10-22T03:46:05Z","signature_id":"4ed47f59bc86333b21b4757e41feb84f761e114b7c5471b7ecc3a3603389e036","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"4df02b86c3017080377e11175c7c07789e1a456c672831bf64f248cd55f61a83","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.976744186046512,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,1,0,0,2,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-24T22:15:18Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24681019eca0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-m6dlx86v7zvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\",\n      \"name\": \"4249c9a2-6a00-4a31-a13c-7b395eeda2a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:21:19.945219062Z\",\n      \"endTime\": \"2026-10-27T21:50:23.791391981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\n            correlation ID: 79f0a27c-81eb-45e3-ba1c-7ed22276e9b1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24681019eca0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-m6dlx86v7zvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\",\n      \"name\": \"4249c9a2-6a00-4a31-a13c-7b395eeda2a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:21:19.945219062Z\",\n      \"endTime\": \"2026-10-27T21:50:23.791391981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9\n            correlation ID: 79f0a27c-81eb-45e3-ba1c-7ed22276e9b1\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-x79lxq7pcp2s/cluster-candidate-4-20-wz9qjx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ea33da4c3e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wz9qjx: failed waiting for cluster=\"cluster-candidate-4-20-wz9qjx\" in resourcegroup=\"rg-candidate-4-20-x79lxq7pcp2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\",\n      \"name\": \"5ef510b2-7fbf-47d3-a252-f868567b1a44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:50:03.68579052Z\",\n      \"endTime\": \"2026-10-27T17:19:03.174752422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\n            correlation ID: 0794d196-09b5-4818-a7ca-7f5fb13bca4d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-x79lxq7pcp2s/cluster-candidate-4-20-wz9qjx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ea33da4c3e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wz9qjx: failed waiting for cluster=\"cluster-candidate-4-20-wz9qjx\" in resourcegroup=\"rg-candidate-4-20-x79lxq7pcp2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\",\n      \"name\": \"5ef510b2-7fbf-47d3-a252-f868567b1a44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T16:50:03.68579052Z\",\n      \"endTime\": \"2026-10-27T17:19:03.174752422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44\n            correlation ID: 0794d196-09b5-4818-a7ca-7f5fb13bca4d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-njchjb\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2294f905a340\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-njchjb: failed waiting for cluster=\"cluster-zstream-4-22-njchjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-5bksdkff6tvq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\",\n      \"name\": \"b7f42e30-e443-47e9-982e-3fe8abc06d9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T22:19:20.094796575Z\",\n      \"endTime\": \"2026-10-24T22:48:29.265123983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\n            correlation ID: c6c76fd1-9a00-45d4-a377-4b7ed8ffb192\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-njchjb\" with version 4.22.18 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2294f905a340\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-njchjb: failed waiting for cluster=\"cluster-zstream-4-22-njchjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-5bksdkff6tvq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\",\n      \"name\": \"b7f42e30-e443-47e9-982e-3fe8abc06d9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T22:19:20.094796575Z\",\n      \"endTime\": \"2026-10-24T22:48:29.265123983Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a\n            correlation ID: c6c76fd1-9a00-45d4-a377-4b7ed8ffb192\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2115186296967139328","occurred_at":"2026-10-27T20:58:06Z","signature_id":"9bf64bab7bd7b09d6d385dd51110a97147d0a8ab7a3ffb3ee7f8b0a43be24676","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2115118782362947584","occurred_at":"2026-10-27T16:29:49Z","signature_id":"591299a6155c115749a9c57afab02aeac1e70aa273d0348576c446c05bcedc8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114118563634614272","occurred_at":"2026-10-24T22:15:18Z","signature_id":"a302bb8010a6d25229c3fd7cf08785413c9415deba7557c8c009262bfec73509","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"35f76f69751e6be795e750bc8688e506149495611433b9eeef712318ba1e0147","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.976744186046512,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["PROD"],"trend_counts":[0,0,1,1,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T04:35:45Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x248e56e12588\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x248e56f14360\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x248e5757aa00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x248e57018e60\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x248e56e12588\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x248e56f14360\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x248e5757aa00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x248e57018e60\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x22d4bdfbc060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x22d4bdf9c1a0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x22d4bdee0f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x22d4bdf9da20\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x22d4bdfbc060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x22d4bdf9c1a0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x22d4bdee0f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x22d4bdf9da20\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x1b9fbf4865b8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1b9fbd955860\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x1b9fbd298140\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x1b9fbe030640\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x1b9fbf4865b8\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1b9fbd955860\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x1b9fbd298140\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x1b9fbe030640\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114214308597469184","occurred_at":"2026-10-25T04:35:45Z","signature_id":"d67ff84dbdc57ce1c070c25b4715de2b3adc2b97c517e1ea3ab71e89571a78e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114118563634614272","occurred_at":"2026-10-24T22:15:18Z","signature_id":"6d6674d8c76b6ba41bbd1f09b2148d963cc126b36b3cf0534e4bc9d68bcab0a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113565541661675520","occurred_at":"2026-10-23T09:37:46Z","signature_id":"f9a18ca69cf4bb95a25d5025c00bc51154f09360293543f668a6db70b9c91d80","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f8a1cfd7bf1a0bd0e82317121262b87bf51fd3aef00a088f1d465c1acea348e3","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-5cjwmbnpxtmc\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.651162790697675,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T07:13:02Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x13a2362ee210\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-5cjwmbnpxtmc\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-5cjwmbnpxtmc\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x13a235e7d850\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x13a2362ee210\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-5cjwmbnpxtmc\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-5cjwmbnpxtmc\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x13a235e7d850\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xa204c386ab0\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-zhz5sxs2r4gt\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xa204b336550\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xa204c386ab0\u003e: \n    failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"acr-pull\\\" in resourcegroup=\\\"acr-pull-zhz5sxs2r4gt\\\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xa204b336550\u003e{\n                s: \"timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2115045747266162688","occurred_at":"2026-10-27T11:39:36Z","signature_id":"1891a29ab5c313967803e3f1bb5cda4edbad4544a1c1644ab1374f58167df64d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114253887920672768","occurred_at":"2026-10-25T07:13:02Z","signature_id":"3530be730cc90660f3f8d03767a868ff1c81d8a0d4db32dd7c61c6a501184ac1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d0f38b59cdb73fe399f095f28adb5d199b8f8fcb7ad873da659c762bd388edba","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T22:15:18Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f57e0004820\u003e: \n    DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3f57dfc6e000\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f57e0004820\u003e: \n    DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3f57dfc6e000\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114118563634614272","occurred_at":"2026-10-24T22:15:18Z","signature_id":"44c8a4ef2c272f4d382f831b9cc84199c2c0fd088544dedc6a0b97aaae3012f7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"3511130a996ff692bc667453c45bd18ba3686849bbb57521e9227fce4426d0d6","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-gt2zw","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T16:40:04Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x7f30d39c620\u003e: \n    DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x7f30d39c600\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x7f30d39c620\u003e: \n    DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x7f30d39c600\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114034199856943104","occurred_at":"2026-10-24T16:40:04Z","signature_id":"73fdea9d0029dbcb11f8faa09d594609d99f6203ae3629bf71437a0675c862c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ee49f58f2d05d626d8e00c13e8e56a93e16bd2c483e3b4ec1f4b514adb0e5a55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Ple... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T03:02:26Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-1-fjj5cf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b484fe8ea00\u003e: \n    failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-1-fjj5cf\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\",\n      \"name\": \"cf1ae438-bf01-4f86-8ab5-784122b7cb5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T03:36:53.315230765Z\",\n      \"endTime\": \"2026-10-24T03:56:02.496643822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j: Failed: interfaces failed to create or update. err: failed to get existing resource rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/providers/Microsoft.Network/networkInterfaces/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Ple... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\n            correlation ID: 3b16cd17-d78f-4b89-a83b-76ef2e36cf89\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-1-fjj5cf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b484fe8ea00\u003e: \n    failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-1-fjj5cf\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\",\n      \"name\": \"cf1ae438-bf01-4f86-8ab5-784122b7cb5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T03:36:53.315230765Z\",\n      \"endTime\": \"2026-10-24T03:56:02.496643822Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j: Failed: interfaces failed to create or update. err: failed to get existing resource rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/providers/Microsoft.Network/networkInterfaces/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Ple... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a\n            correlation ID: 3b16cd17-d78f-4b89-a83b-76ef2e36cf89\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113828436664389632","occurred_at":"2026-10-24T03:02:26Z","signature_id":"592a40423d8b93557d7d4b0a395f2fe0758660664ebe176df6f7768b6f62a74e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8b157df38af3252928b0d0c8ebb12f675589f14b62d8751da69093589eb2fca4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[2,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T04:35:45Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cdcfbca5680\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-8vtwfvfssx9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\",\n      \"name\": \"907ac069-8701-4021-b410-cf10181345c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T05:14:31.336281353Z\",\n      \"endTime\": \"2026-10-25T05:33:33.578871475Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cdcfbca5680\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-8vtwfvfssx9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\",\n      \"name\": \"907ac069-8701-4021-b410-cf10181345c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T05:14:31.336281353Z\",\n      \"endTime\": \"2026-10-25T05:33:33.578871475Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0\n            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114214308597469184","occurred_at":"2026-10-25T04:35:45Z","signature_id":"bd9e8e0cae4c284361718774eca639e561433edd1d045672abb6f55be42cb28b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"67b50109702bf7d33ec41a7f197070b9f80a5774c8314d0444fba9554c957618","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T16:40:04Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9134b719420\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c2zb7x4tvbjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\",\n      \"name\": \"2d4481c2-5c8d-4e2e-a526-9b051ca6139f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T17:16:34.851278228Z\",\n      \"endTime\": \"2026-10-24T17:45:35.221810696Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\n            correlation ID: 4e6d7a8e-ea3e-4773-abae-818d5133da86\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9134b719420\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-c2zb7x4tvbjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\",\n      \"name\": \"2d4481c2-5c8d-4e2e-a526-9b051ca6139f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T17:16:34.851278228Z\",\n      \"endTime\": \"2026-10-24T17:45:35.221810696Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f\n            correlation ID: 4e6d7a8e-ea3e-4773-abae-818d5133da86\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114034199856943104","occurred_at":"2026-10-24T16:40:04Z","signature_id":"7de8e251b0662ec5525289aa588b2d853800f034c51cea61d098561710ef5052","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"fb4f6400d6d7493af89fac0c7c4d2b0db40240d0c1a506d29ddbd9dd20ae7874","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T14:13:25Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e6e8804d3c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-m5vrcvff2ctf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\",\n      \"name\": \"75b8b1f9-f718-45b2-b1ee-a3009f824cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:53:42.149827787Z\",\n      \"endTime\": \"2026-10-22T15:12:50.02702561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\n            correlation ID: e990e88e-110c-474c-9878-aa84f73234da\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e6e8804d3c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-m5vrcvff2ctf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\",\n      \"name\": \"75b8b1f9-f718-45b2-b1ee-a3009f824cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T14:53:42.149827787Z\",\n      \"endTime\": \"2026-10-22T15:12:50.02702561Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8\n            correlation ID: e990e88e-110c-474c-9878-aa84f73234da\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113272516096036864","occurred_at":"2026-10-22T14:13:25Z","signature_id":"ce2be9d56c79e39c69b76958f0e15893b7706328523af921efd83797b0c81835","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"b94e9ac8495cf32cf3307d63c94c1a78ff8c4978a6d0ece0301068a33865acce","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptACIProvisioningTimeout; detail message The supporting resource: 'Azure Container Instance' did not provision during the expected time. Correlation Id: \u003cuuid\u003e. Please try again later, if the issue persists contact technical support for further investigation.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Cluster Etcd Managed HSM Encryption should create a cluster with Managed HSM keyVaultType and rotate the etcd KMS key","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T14:13:25Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Cluster Etcd Managed HSM Encryption should create a cluster with Managed HSM keyVaultType and rotate the etcd KMS key","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_etcd_mhsm.go:106]: failed to deploy and activate Managed HSM\nUnexpected error:\n    \u003c*fmt.wrapError | 0x563a226e760\u003e: \n    failed waiting for deployment \"mhsm-deploy-etcd-mhsm\" in resourcegroup=\"etcd-mhsm-blgzqwztp2gh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentScriptACIProvisioningTimeout\\\",\\r\\n        \\\"message\\\": \\\"The supporting resource: 'Azure Container Instance' did not provision during the expected time. Correlation Id: c1661afd-ea85-487b-8e90-65a3f8664855. Please try again later, if the issue persists contact technical support for further investigation.\\\"\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271\n            correlation ID: 989b1e9c-002b-494d-ae3b-825cc5f4adf2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_etcd_mhsm.go:106]: failed to deploy and activate Managed HSM\nUnexpected error:\n    \u003c*fmt.wrapError | 0x563a226e760\u003e: \n    failed waiting for deployment \"mhsm-deploy-etcd-mhsm\" in resourcegroup=\"etcd-mhsm-blgzqwztp2gh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentScriptACIProvisioningTimeout\\\",\\r\\n        \\\"message\\\": \\\"The supporting resource: 'Azure Container Instance' did not provision during the expected time. Correlation Id: c1661afd-ea85-487b-8e90-65a3f8664855. Please try again later, if the issue persists contact technical support for further investigation.\\\"\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271\n            correlation ID: 989b1e9c-002b-494d-ae3b-825cc5f4adf2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113272516096036864","occurred_at":"2026-10-22T14:13:25Z","signature_id":"df2ccb9cac46575170ee8c3161e55da965625384c8157f9818ff453d82660f36","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a943d2907074ed1f6f29dea5f81c6cf99bc3a7668efe0e4f83b02f9f453e38ff","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-c2hldx' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T07:13:02Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-g8n765\"\nUnexpected error:\n    \u003c*errors.joinError | 0x254aca410498\u003e: \n    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record \"*.e2e-dns-c2hldx\" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-g8n765\"\nUnexpected error:\n    \u003c*errors.joinError | 0x254aca410498\u003e: \n    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record \"*.e2e-dns-c2hldx\" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx\n            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114253887920672768","occurred_at":"2026-10-25T07:13:02Z","signature_id":"93ba40c9e936f2e4832deda7d0989a60e6d92d450dd6a05b400d12ed625527fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ade586e4ff4d1a0ecc30863beb272f72be936c20b795ba2fa8184e6ebc881e69","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-fn44p6' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-rf46z8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3ac741d4138\u003e: \n    VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.001s: DNS A record \"*.e2e-dns-fn44p6\" not yet present in zone rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fn44p6' does not exist in resource group 'rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6\n            correlation ID: bf6f75cd-f285-4895-ad70-7d62eff4cba2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster \"cluster-candidate-4-21-rf46z8\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3ac741d4138\u003e: \n    VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.001s: DNS A record \"*.e2e-dns-fn44p6\" not yet present in zone rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-fn44p6' does not exist in resource group 'rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6\n            correlation ID: bf6f75cd-f285-4895-ad70-7d62eff4cba2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114679321305026560","occurred_at":"2026-10-26T11:23:33Z","signature_id":"82deb4990b60bbf97dd2450aa9bc4e0df1afaf90026b1c2dc14bf83adb32f3cb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"1e4dfca010d42443afaf45bb97ca117493c5e3aa47ab2e1a79cdf35dff399c0c","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-jgd8x5' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T11:32:35Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-652kdm\"\nUnexpected error:\n    \u003c*errors.joinError | 0x282c0f144570\u003e: \n    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record \"*.e2e-dns-jgd8x5\" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n            correlation ID: a77d314a-799e-4560-8840-6738610cf0af\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-652kdm\"\nUnexpected error:\n    \u003c*errors.joinError | 0x282c0f144570\u003e: \n    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record \"*.e2e-dns-jgd8x5\" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5\n            correlation ID: a77d314a-799e-4560-8840-6738610cf0af\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114319204998975488","occurred_at":"2026-10-25T11:32:35Z","signature_id":"83028b765d97711c40f0cfa5de1d458e748cb143abe43b894f475efa5973d496","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a969ecfcb2e0bc52f55dd728d038e5519456118b2d8e50c23ca4ff2b5d8d15cb","failure_pattern":"ERROR CODE: NotFound; detail message The resource record '*.e2e-dns-rg99mn' does not exist in resource group '\u003cresource-group\u003e' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.; provider Microsoft.Network","search_query":"ERROR CODE: NotFound","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T13:11:36Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-4fhvgp\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3188fb29e5a0\u003e: \n    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record \"*.e2e-dns-rg99mn\" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-4fhvgp\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3188fb29e5a0\u003e: \n    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record \"*.e2e-dns-rg99mn\" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: NotFound\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"NotFound\",\n      \"message\": \"The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn\n            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2114344123895910400","occurred_at":"2026-10-25T13:11:36Z","signature_id":"e2f09681e2ed91758f8d8ecaa88afe2e6d3f1537b4d202fd8b86eb96323adc53","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"fd502680ec0e20831c7a9343582e93a9e74519fa4b943687d22510f1e9b188c1","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T03:02:26Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x11d151799220\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x11d151799220\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113828436664389632","occurred_at":"2026-10-24T03:02:26Z","signature_id":"b956b788ed25c49bf2dc7666edf7a36bcdc87c0a53543bb1dc045942b117250a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"25d52162053e586fcf0f8b1f80f5bd0904e6a9593e2a7e8aea650a9440b7f090","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cghxqb\"\nUnexpected error:\n    \u003c*errors.joinError | 0x315939ea25b8\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x31593941d8c0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x31593941d880\u003e{\n                    msg: \"failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x31593941d840\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x31593a26f680\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster \"cluster-candidate-4-22-cghxqb\"\nUnexpected error:\n    \u003c*errors.joinError | 0x315939ea25b8\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x31593941d8c0\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x31593941d880\u003e{\n                    msg: \"failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x31593941d840\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\": nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x31593a26f680\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2115186296967139328","occurred_at":"2026-10-27T20:58:06Z","signature_id":"86b73281a2208a753a1d5f16b0e817bfb68b9e12d72d843225f3aa4e44ea0770","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"7f5bae270b392d43d5d38e020472dadca9f4bd4dfd9b43e64ef7a484779b65a5","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T12:28:07Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e28491f1160\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e28491f1160\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113608404747423744","occurred_at":"2026-10-23T12:28:07Z","signature_id":"0f2d86e05d0cc2787fc09c17fb75f78487dd7070920c395899d0bbe661d9883b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"3b856f6a658ebfd5153a95a75d2264f55ac5d0e2e542d440fa5e2b241bdeb285","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-lrpv2qd\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T09:37:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-4x46vw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b1760358220\u003e: \n    console fixture namespace \"console-lrpv2qd\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-lrpv2qd\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-lrpv2qd\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x3b1760b23ba0\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-4x46vw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3b1760358220\u003e: \n    console fixture namespace \"console-lrpv2qd\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-lrpv2qd\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-lrpv2qd\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-zhbqp container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x3b1760b23ba0\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113565541661675520","occurred_at":"2026-10-23T09:37:46Z","signature_id":"732d5be336bcb1714448223e4f49a5b8b72c38fa5b2289aaa9dcfbf96cbdeaee","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"5a5158d8b719790176f7c22c4c6ca69a3ec44f23608973f87b7c40193f0d3ea9","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-mr8rvbh\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T08:38:55Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-5-0-6rxg4b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12de498cec20\u003e: \n    console fixture namespace \"console-mr8rvbh\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-mr8rvbh\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-mr8rvbh\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x12de4a14f120\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-5-0-6rxg4b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12de498cec20\u003e: \n    console fixture namespace \"console-mr8rvbh\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-mr8rvbh\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-mr8rvbh\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-zhr7j container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x12de4a14f120\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113188336309899264","occurred_at":"2026-10-22T08:38:55Z","signature_id":"935354fd613db5a87f82c12d280e6c6e837f59475cdae991d9c2503b0a89875d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"c82ecb33cdc9e243c13e87455cd57bd79deaaa9db83c3fd5331fc5fc41e86b9d","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-n2bkhw2\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T09:37:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-5-1-qx8d44\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39e6141ab040\u003e: \n    console fixture namespace \"console-n2bkhw2\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-n2bkhw2\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-n2bkhw2\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x39e614e48260\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-5-1-qx8d44\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39e6141ab040\u003e: \n    console fixture namespace \"console-n2bkhw2\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-n2bkhw2\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-n2bkhw2\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-nrx7d container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x39e614e48260\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113565541661675520","occurred_at":"2026-10-23T09:37:46Z","signature_id":"42ee2132cd101ce82f1bc410cae42ba902a7347abeb11fa8f07b5760c9535aa1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"298033e1b3f566b419dbeaf66b89e5d6c9f9fdbd8e4f432fd529054b1a45fffa","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-r7gptrj\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T09:37:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-5-0-glc2bs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d9cfabb2a80\u003e: \n    console fixture namespace \"console-r7gptrj\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-r7gptrj\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-r7gptrj\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x2d9cfac24920\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster \"cluster-candidate-5-0-glc2bs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d9cfabb2a80\u003e: \n    console fixture namespace \"console-r7gptrj\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-r7gptrj\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-r7gptrj\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-77657bcfc5-mwmw6 container=\\\"dex\\\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x2d9cfac24920\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113565541661675520","occurred_at":"2026-10-23T09:37:46Z","signature_id":"18bb74972d5252a1876fc2b0fee927d27c6abe5c2d2bb5dba2982a49bdbf5ccd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"54ef2eed365c9e873383a8732cab276546d23c011cd46df77acb7f8885385547","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-xzp4sq4\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T10:51:53Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-8dwfln\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22821099b6a0\u003e: \n    console fixture namespace \"console-xzp4sq4\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-xzp4sq4\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-xzp4sq4\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x228210e20900\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-8dwfln\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22821099b6a0\u003e: \n    console fixture namespace \"console-xzp4sq4\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-xzp4sq4\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-xzp4sq4\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-swzhr container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x228210e20900\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113221801671659520","occurred_at":"2026-10-22T10:51:53Z","signature_id":"9cc7bc1a16db47f1e81a1d8619fde17cf573128e92045b0e3d5d9c5a99bf3fd2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"38fc1d1a69ad57bdc0afd589224269cd4be9e1e69579f3ad54e3d7dc78ffbaf5","failure_pattern":"console fixture namespace \"\u003cnamespace\u003e\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex","search_query":"console fixture namespace \"console-z2wqbfw\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\"Running\" reason=\"\"","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.3255813953488373,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T03:46:05Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-9js6f6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10354e044ea0\u003e: \n    console fixture namespace \"console-z2wqbfw\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-z2wqbfw\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-z2wqbfw\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x10354ed78880\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster \"cluster-candidate-4-20-9js6f6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10354e044ea0\u003e: \n    console fixture namespace \"console-z2wqbfw\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\"Running\" reason=\"\"\n    {\n        msg: \"console fixture namespace \\\"console-z2wqbfw\\\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\\\"console-z2wqbfw\\\"; Deployment/dex condition=\\\"Available\\\" status=\\\"True\\\" reason=\\\"MinimumReplicasAvailable\\\"; Deployment/dex condition=\\\"Progressing\\\" status=\\\"True\\\" reason=\\\"NewReplicaSetAvailable\\\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\\\"ContainersReady\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"Initialized\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"PodReadyToStartContainers\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"PodScheduled\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx condition=\\\"Ready\\\" status=\\\"True\\\" reason=\\\"\\\"; Pod/dex-78f4dcc67d-6mptx container=\\\"dex\\\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\\\"Running\\\" reason=\\\"\\\"\",\n        err: \u003c*fmt.wrapError | 0x10354ed78880\u003e{\n            msg: \"configure Console ExternalAuth through RP: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel-all/2113114644393496576","occurred_at":"2026-10-22T03:46:05Z","signature_id":"da226fa88ddd3ebd7fca5f82d2dd05b81889748bd96226703807023dc9f737d0","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":76,"failed_runs":31,"raw_occurrences":318,"matched_occurrences":311,"runs_affected":24},"rows":[{"environment":"prod","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":31,"runs_affected":5,"failed_runs":5,"run_impact_percent":6.578947368421052,"anchor_occurrences":31,"after_last_push_count":42,"also_in":["DEV","INT","STG"],"trend_counts":[0,10,0,0,0,0,2,19],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-12","2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-s7k2sd\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x297062e2e820\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster=\"cp-ystream-upgrade-4-23-s7k2sd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-np7xjvwc8xct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n      \"name\": \"5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:32.825742556Z\",\n      \"endTime\": \"2026-10-28T20:41:35.067951058Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n            correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-s7k2sd\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x297062e2e820\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster=\"cp-ystream-upgrade-4-23-s7k2sd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-np7xjvwc8xct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n      \"name\": \"5d9c960a-8f4b-476d-bd5b-132c0ce76460\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:32.825742556Z\",\n      \"endTime\": \"2026-10-28T20:41:35.067951058Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460\n            correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10164f31ca00\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p669mb2thnqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n      \"name\": \"f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:27:58.648367476Z\",\n      \"endTime\": \"2026-10-28T19:57:03.816857754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n            correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10164f31ca00\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p669mb2thnqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n      \"name\": \"f26dc7e7-f380-4b37-87cb-db720fa2be9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:27:58.648367476Z\",\n      \"endTime\": \"2026-10-28T19:57:03.816857754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b\n            correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lrsdfl\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c350f225e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lrsdfl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\",\n      \"name\": \"034eed87-a977-47cf-8ff5-2b8292aa4616\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:34.208698264Z\",\n      \"endTime\": \"2026-10-28T20:41:35.072616907Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n            correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lrsdfl\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c350f225e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lrsdfl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\",\n      \"name\": \"034eed87-a977-47cf-8ff5-2b8292aa4616\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:34.208698264Z\",\n      \"endTime\": \"2026-10-28T20:41:35.072616907Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616\n            correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"25962cc73cb06a1728ac605aa1f4b06859ed88a0569ae6115f498a1cbc7e4539","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"5b7dc99f064f8216ed626ab763d7c2a790998f03c03f0d31c47ee72a9d07d7aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"61abbe86523cd0cbbe34f1a7d4e1334ee3a9f32d67628855d0ff113bf0b2c7b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"a580fa2d1a60bc0f4d0f383e974139db9c4e6ed9723aac1cfd38205ae56a2708","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ad80f1e1dfb5cb1f986c967a0119074ab300822bb8d46340e61875c7b780b23e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"bec54b9ff9b28304b3228ba00ce3498e04da34d0d0ff7192696df423f2a05e7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"008b7fa14234b7fd53ce92c2a67a34bd489ce71e51f2b07003c764af8afab38b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"4eacc122e24dff876da529f641ce2f8e88118e1c180fe85bb99ae5977c4eeb02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"57a9b4fb81f9a741827a0d6649ef52cee527324f4edfd50bc19545467e6ca18f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"7a2a1051e2c8780e79f7d927cf834797fda62d7186cdfde4d3845eb3ca5413bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"88fea14a4d2f8221bdce2a4cdad6016802ef185095cab4c049686833df5861ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"94c0703b782d06837b72402fee70c4cd576d6ab0dd07de6e02fc57033e76a4a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"a7147f358a6b4c690915b8e784595bbb5c2a8d551bf2578e9e4fcc175434ec22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"aaae8a3dd4bb9cf02e846a792df83408b7d48c787c029d118f706904b2dfba5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"cbc0103ee5a417fd440bec6956285feaa5bda8eafcb0c41d737a626acc594ca2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"d13b782f0b69e1d4260e04e9bf10320e2c7adb462218158530bd960678b2de8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"f16235cd783938327ef4a9a477b7e7b8c1f5b1782a0f3686b7101daa1e644956","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"fa11fc83ee685e2b0b8139fe2a6ad2007ed47287825b00289738db00e3401559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"fb1fb5f7ad13ff275d52922aefa84c5111fdc94b3d6726b209d19396eb7f5dbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115132439494922240","occurred_at":"2026-10-27T17:13:43Z","signature_id":"40ae0efd6de3572a94cab845240bbcef9b9dfc8cfff4b3e416542ee5887b4771","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115079131212288000","occurred_at":"2026-10-27T13:52:15Z","signature_id":"28c064953b5e747092df5155eb91c076c0e4eab8fc509418e936be2add3f6b73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00c7da51be8e89ce15ce8e10590e25d85b2282252d307b006276577d947b9e86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"01479cc8260099169681c3c9aac7e1601c47f3919a6c41c57610a9af6d7dfa91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"2ca8d85c3dc2b47b78c6a24945b274f76b13627c62c8a5c01d3ebc927fad0021","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"541341a5ff6a403f156c133d1168366027c4c77d725975906770d24cc82a1c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8443649e9083d1ab2db3cf3952a87eb977678118008a762dcd368311d2aaac93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b311e9b9b11e8b11ec8d0f86f4d29406e7d83165875e0863fb9281515f0c7523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ba521c2c19919a5caa070da355f36879cb81eb48d013d05ff609671940da6a95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"c0bc7ce2f8badc2267d7a65ad1002f4d46160b2c3b4087c1135728e23c2b816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e4e0808ae7571f998df5b39264b674c05cb041138f2cb2532613af052179af56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f7d5f9aaa39bc4f25aca130a942f0670850a3d28be7139dc0a9df1e58b8e9577","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":28,"runs_affected":6,"failed_runs":6,"run_impact_percent":7.894736842105263,"anchor_occurrences":28,"after_last_push_count":34,"also_in":["DEV","STG"],"trend_counts":[0,5,10,5,0,5,3,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-lkx8t5\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3306da82b390\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-lkx8t5\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3306da82b390\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xdf906826d40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xdf906826d40\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3454a17e2bc0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3454a17e2bc0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2114899906912063488","occurred_at":"2026-10-27T02:00:05Z","signature_id":"706d495a0a2301a008f74156eee746fb4dcedf7d96ad556dbc57056c56f128b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2114899906912063488","occurred_at":"2026-10-27T02:00:05Z","signature_id":"b1d057f63ee2b18cae70686f3aeef3fcdbae4dd7cf706e29f2e4659fcabf80a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2114899906912063488","occurred_at":"2026-10-27T02:00:05Z","signature_id":"dad14ce21a2e8e9a6601ca209a6a53ad3686cf903b1fa5fa19f9fd00f1ae154c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"0bc78efff4846201b543d5e289f2d440179be46611a47fe37649d4bbf599d4ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"6ec466c8b8b8d92cdf22928a55a50e11001084bb1c81544ed59b37e165d909bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"9e52f9370f3e00a595c6ad24f7c126d79e53755a58efc81043e759a5fc82e67a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"b485a1868f41ef68d4a2ca6663bb793f19a751b4c126bd42a71316a81defe5f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"d6ded71bf0676f847857aaaabe8b49a639798e3e27d8ac55c25535b0a94ed186","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"33db1c3a753f86de5437aeea4ce94aea50eff6efcc9e24481fd3b5a5db91355d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"3f2b2137f83a3269a387de589f4ec90449d00621fe948040b817efb69653752e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"6464f6e3e62a79a6ecad40d9da5fa9f7a3ca4e1c274f987a91f46ad0ef5805f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"6aad85da11b23d51d01d7048ab2fcb7dd17e96a24595d2c070d12b0e2654fa14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"8d94601ad9e8035e94766bfdd3ff32bdad6a27008af0242092dd096fe5ae318d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"a9d0787a49a9434b575f780199c840b15cab7a7f5c8097f81f3e40e8e6452e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"bd72849f8bfb61f3950e07a620fb8b9bddc49d0f09058e4f6ff7f7eef02627f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"c3d8588a7330141641207b393ef0b056b3f565ec62f5715a57756d5035d5fa24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"e444cd22aa468d73f20a8d2ff0e73fd525cfd2b1f33d4eab9b629bfe695606fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"f8ad7afc2b67ed6f03b36c92fec6e7ed4f59bd175316c494010190cf2fe6a1df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"083741f16af6bde01a15c32f163b36e9e075c1b8a96f107ca16e1a1830aa7995","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"2d6ae229654176a356f5c870d83add62836595109dbb7a96870fa8c8c6a4e33b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"4a987599543343a0a61051617564a6330ab4d61da2e4fe37b0c8507e51dec2dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8ae7224beef582a60fc95922521582cea88650e7efa805e02a16af12f707e016","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"951e3efbb76f026e0f9c0427659ee8ef966b4d7a1594708f3b87a3b80dade019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"28f82d6598d8560878485b881cedade2766f6f99a7665f52a5642ec9863e8f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"554520898e3bf65aa33d08c67763b5d08098c2dea88239fbc79acfa28b30250c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"91144e10f499ef01ce91bc27b79924e73fc51fedcf3381a2242dc3972585a568","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"ad2974ff484501a74259c346bde4f86a21cbaef053d5d94585993a7186adc875","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"d6194b621062ac0dc7417d99a6662052b73f0727482224784de827ab3d686809","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":23,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":23,"after_last_push_count":26,"also_in":["DEV"],"trend_counts":[0,9,0,0,0,0,0,14],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-qtj68sbhkffm/cluster-candidate-4-20-hxsx6r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15a8256fc4a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-hxsx6r: failed waiting for cluster=\"cluster-candidate-4-20-hxsx6r\" in resourcegroup=\"rg-candidate-4-20-qtj68sbhkffm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n      \"name\": \"93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:04:15.708912895Z\",\n      \"endTime\": \"2026-10-28T20:33:16.216619801Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, olm-operator, cluster-storage-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, packageserver, catalog-operator, cluster-policy-controller, cluster-version-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n            correlation ID: b596877d-09f6-4478-a504-ebb349eebdd1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-qtj68sbhkffm/cluster-candidate-4-20-hxsx6r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15a8256fc4a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-hxsx6r: failed waiting for cluster=\"cluster-candidate-4-20-hxsx6r\" in resourcegroup=\"rg-candidate-4-20-qtj68sbhkffm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n      \"name\": \"93a53e80-e0f1-4f90-b3ef-4df9d072a238\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:04:15.708912895Z\",\n      \"endTime\": \"2026-10-28T20:33:16.216619801Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-controller-manager, kube-scheduler, hosted-cluster-config-operator, machine-approver, control-plane-pki-operator, azure-cloud-controller-manager, konnectivity-agent, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, olm-operator, cluster-storage-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, dns-operator, ingress-operator, cluster-image-registry-operator, packageserver, catalog-operator, cluster-policy-controller, cluster-version-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/93a53e80-e0f1-4f90-b3ef-4df9d072a238\n            correlation ID: b596877d-09f6-4478-a504-ebb349eebdd1\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd149dec0ae0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6nnwct2hkjrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n      \"name\": \"89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:20.936404311Z\",\n      \"endTime\": \"2026-10-28T19:53:23.678300779Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-version-operator, openshift-controller-manager, kube-storage-version-migrator, catalog-operator, azure-cloud-controller-manager, dns-operator, redhat-operators-catalog, cluster-storage-operator, control-plane-pki-operator, openshift-route-controller-manager, hosted-cluster-config-operator, packageserver, konnectivity-agent, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, certified-operators-catalog, machine-approver, cluster-network-operator, ingress-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n            correlation ID: c78028a7-799f-4fe4-8afb-462431828f6c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd149dec0ae0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-6nnwct2hkjrk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n      \"name\": \"89ea4fd7-6047-49a5-afc1-c0d8ef75994d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:20.936404311Z\",\n      \"endTime\": \"2026-10-28T19:53:23.678300779Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-version-operator, openshift-controller-manager, kube-storage-version-migrator, catalog-operator, azure-cloud-controller-manager, dns-operator, redhat-operators-catalog, cluster-storage-operator, control-plane-pki-operator, openshift-route-controller-manager, hosted-cluster-config-operator, packageserver, konnectivity-agent, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, certified-operators-catalog, machine-approver, cluster-network-operator, ingress-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89ea4fd7-6047-49a5-afc1-c0d8ef75994d\n            correlation ID: c78028a7-799f-4fe4-8afb-462431828f6c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b3d00e688d0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n      \"name\": \"e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:59:31.361013478Z\",\n      \"endTime\": \"2026-10-28T21:28:40.621642045Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, packageserver, dns-operator, ingress-operator, olm-operator, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, control-plane-pki-operator, openshift-route-controller-manager, konnectivity-agent, redhat-marketplace-catalog, community-operators-catalog, cluster-storage-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n            correlation ID: caf44bbd-ef58-4e77-8074-5770ca1391e6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b3d00e688d0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n      \"name\": \"e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:59:31.361013478Z\",\n      \"endTime\": \"2026-10-28T21:28:40.621642045Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, packageserver, dns-operator, ingress-operator, olm-operator, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, redhat-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, control-plane-pki-operator, openshift-route-controller-manager, konnectivity-agent, redhat-marketplace-catalog, community-operators-catalog, cluster-storage-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e764f65a-623b-45b4-b7fa-b7b1ef90a8a0\n            correlation ID: caf44bbd-ef58-4e77-8074-5770ca1391e6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"1b112c4de17bdc1ed6df2e4af8db0e4c69a28c7d216afb2f5acd996a62fa50e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"36ab0f864c0e4e4ebea7cdce2d76b37ddec10828ae4d6d3f57c35bc6ef336362","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"4d63e7a6f84ed9ae74bd797e00b40ae72a6ac8953997ae0f91886ae1595029e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"5a2acf40c621f89bf24df745acf1ec5ca1bec8954c5e1db840e0c4501fae6ee4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"5f6d04b8f2b2c5c630e47972114cb8f68ab1ab545e7516b6f7ff6f5109fd3680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"d347788cb0265beb929b34ab787405bf42d848f46684380adad1dd8be49a95ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"088a3092facb9e749cf94ca46a9c078302ab260e25eb1075b25fd77f9f9504a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"0c91a1ef66600b81d08ebe7871bd904a4fa12212cb8e7cf459bab1c021971b15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"17b8a61ee96a7fb815efb7d2c1a5d6db362fee25a96c755eb6f05ec793687672","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"4da04c40a2d7ba226fa1cf6a9bb0dc8d5bdc90ab92882a6f7e685554c77499e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"64a40c08ccfc6a3b9dc40685da0aca81380375f2e59b1d6dbc879b99d641bf1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"6a88c826b274fcdb6aada26d363a44e43a7b4641e212999dd5097ed4710a8547","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"8b0c818dceeff02dbaca5d871463efcf6a41f805938018d938f9ad5a462f8dce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"a968cd687e32398c82a203a778a7c7df20f60576fe20d79ea6c3c7b23c10b036","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"012587d7d4630c81e42093fd6ab2c1257951217e2525d7784fd89a2e89112676","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"1e540a6273c70580b8d0a813a5ac2b3184c764a6be285464f22624d92e6da8b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3869c78574390577fcc299556aef741fbec4d0cca2bc649c0c2a2e1cd5f36f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3dd0847594fe9934ecaef404cfd5b2e0d30bdd83e3bdd0629d37df342bb8fe04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7a1cec7cd36f34b86775ec4e998ac79d4f3902ec309e0df7dd447a697e1ef501","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"884359bc37b7b0f624f652c6e3ab6489f45271d6a5f2eaf45cf5d681cd51b93c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"940d1146406ccc6a3297d550ec44a0b5d1a43287589036549f13183fd87e514e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d15e0ee9a765947a51994971a21dbf7f7f640ff3a8e13f8d501e97316301c1e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f052c83930df14da60956d205c94d7b6903bdbd091c343818e9c8edabac6c7c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"94cd5d525f06d1186ecd21cd0e49e732344fcbd0faf84079a573baea2e1b0d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":23,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":23,"after_last_push_count":35,"trend_counts":[0,7,0,0,0,0,0,16],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22868c3a2660\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s6kbbzpqdbx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n      \"name\": \"a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:09.588021132Z\",\n      \"endTime\": \"2026-10-28T19:53:13.758028769Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n            correlation ID: 8f02eb4e-1e36-4dd4-b460-7588fb948e71\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22868c3a2660\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-s6kbbzpqdbx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n      \"name\": \"a285ee7f-e10b-45e8-90d8-9ff129ec2b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:09.588021132Z\",\n      \"endTime\": \"2026-10-28T19:53:13.758028769Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a285ee7f-e10b-45e8-90d8-9ff129ec2b53\n            correlation ID: 8f02eb4e-1e36-4dd4-b460-7588fb948e71\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x263f7a1ecb40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s854pt5lzxq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\",\n      \"name\": \"bc13db95-dad4-4bc2-9929-827244e95d2e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:34.589492538Z\",\n      \"endTime\": \"2026-10-28T20:43:34.88292258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n            correlation ID: da1e3131-8ea4-44cd-87a3-857957002869\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x263f7a1ecb40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s854pt5lzxq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\",\n      \"name\": \"bc13db95-dad4-4bc2-9929-827244e95d2e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:34.589492538Z\",\n      \"endTime\": \"2026-10-28T20:43:34.88292258Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bc13db95-dad4-4bc2-9929-827244e95d2e\n            correlation ID: da1e3131-8ea4-44cd-87a3-857957002869\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-clmhlg\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d29037be7e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-clmhlg: failed waiting for cluster=\"cp-ystream-upgrade-5-1-clmhlg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-m7flj58khr4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\",\n      \"name\": \"75f18780-46df-4ca4-b758-e34d2262ebd4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:38.563887566Z\",\n      \"endTime\": \"2026-10-28T20:41:44.890635353Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n            correlation ID: ae6aa7e5-e845-4182-9588-f52bb0a88718\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-clmhlg\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d29037be7e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-clmhlg: failed waiting for cluster=\"cp-ystream-upgrade-5-1-clmhlg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-m7flj58khr4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\",\n      \"name\": \"75f18780-46df-4ca4-b758-e34d2262ebd4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:38.563887566Z\",\n      \"endTime\": \"2026-10-28T20:41:44.890635353Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75f18780-46df-4ca4-b758-e34d2262ebd4\n            correlation ID: ae6aa7e5-e845-4182-9588-f52bb0a88718\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"154d3925be5c012a5113ed89b1b462845af15f57fbfd8e755b22fb15a8b69ce5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"44d9b2ce46d7f92b1a97a2b7006b43cfe018754c0a9d5a038f4bf77df49a7765","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"68b0cfa89285503fbd77f7180bd24bbc791f53e770eaf8d7210f219c10f28a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"762fee71e1d321f9cbabc110e98c51820d49a46e2eb414f6e17c5eff0e5b261d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"90762cacf1fc23e458f3602d5ca3c418a070b1c0b1de0b8c373357dcb930302c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ab061e66e29e6d1c2a90de4abf7a2dc3ca11982db8c3aca1c2d0b21771747f26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"d25cb3f84c24fd62b741353ecaf77155f1270ddfcbac771250ea07b39d1a3825","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ef55f5ef8cc6c413d0c5d55981f0b6ab1b6f752dc03d5d60ed0a2e58fc37935b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"fa5c99ec12e75a03478d289817e89bb3f5f8869f0cee804d3097d4317f930455","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"2407003efa87938c4c66d5a1180adf130e092f7f7f32eaf09748bc884b75abf6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"30ec818b40b7b823a54f8213ce2ad6d08f8cd9991b7021927d7167695678fdc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3d5e4237e6dcfddccbad19ae692ac75d01ca62ee7e2d4940846201edb7aa7425","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"667c970b0bc4106596e97478abc3ff3ace5bb451af5139645277b065301185cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"6ecbcab48c36fbfa7e11bc04e865a63921a9d07e34834d7625911e0fc3cce406","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"91d11f218a6452991755580657a8dab16f1e57a19c912b53be3b78934bbb305d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"b5c0928fd45bfb1b4de0807a52343c6bd769a668574d92535003b3a4c3fa886a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"00e889108d18b2602a9701e81c628e0117c6af2ac6506f48e3dc6377714f2dec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3b522484ec379b7598bbd268e9f04bacd0edc777033a068970a7c75bb7fd3ebc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"51ee050adec069064368b692e2e9f92789ea23eb4ea27242862d06944731e6a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"93b4242fbca78a6804343fa0c780a64cee3b6f7f909e539aa95b885d3208f48b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a705768924b8c91914dfee887fd041226da8b0d2a7c49c71af1ef216422bf744","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"af774aac99c529ad98d882b83b9a1e38d054f96b4ba76e020f5318718f9b0f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d28602bc01604b20a4fd1810051138a3b4d9daa151d24a925a551647d550c3c4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":22,"runs_affected":6,"failed_runs":6,"run_impact_percent":7.894736842105263,"anchor_occurrences":22,"after_last_push_count":22,"also_in":["DEV","STG"],"trend_counts":[0,5,7,5,0,4,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-lbfkz7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x625ce9962c0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x625d05dc8e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-lbfkz7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x625ce9962c0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x625d05dc8e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-6gm2q7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb6fec4b44e0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xb6fec4b44c0\u003e{\n            observation: \u003c*errors.errorString | 0xb6fee7deaa0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xb6fec4b44a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-6gm2q7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xb6fec4b44e0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xb6fec4b44c0\u003e{\n            observation: \u003c*errors.errorString | 0xb6fee7deaa0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xb6fec4b44a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-l72l2j\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8108240e540\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x8108240e520\u003e{\n            observation: \u003c*errors.errorString | 0x81084000ba0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x8108240e500\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-l72l2j\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8108240e540\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x8108240e520\u003e{\n            observation: \u003c*errors.errorString | 0x81084000ba0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x8108240e500\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2114899906912063488","occurred_at":"2026-10-27T02:00:05Z","signature_id":"001e47b2e7765b9a70ffe8f0cee8b56a86fc139e54fd1e21229f45b62a05b093","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"0efb5b8ff49f4444cf0843cd13ba57d7a7c819c35d19f98ba911a4103fbbe88f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"1419cae972bfc0fa8c9c8f0c1403beb1908dba078b179684ac9dc4ba6e7b495d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"1845ddc664b15ea9f03a6db4b34df4fb38db4afd1d0981242acde03b46bfdfcd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"7b5cac0718768715153a920f81ef2fa98163f3bb2a1a0cea82dc5fbc6a09ff32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"48bcb095dab95d4300f51495f479877c48fae6ac937eca31ce9ee317ffaa9c61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"b1dfe79b23dde219591e24160c9becc4dd2e4815fc615b83f459dd71654c29ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"c38d06858d691d9e4c16513cdbf71adc44dfc0e2e5bba3c53da574b0cb0fcaa2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"d12f7ad296c63d2db8d0091c07a0e4b76c50e119a987979054a17bce6c78d0bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","signature_id":"f2369be4ef1db102806ff130f76a044fc5b4c806e03890b5fd3f3eccd39596a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"5124c660676e607323f38b7ee2b1cca5949648ceecce1866b6241aed16b10219","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"f9b8e244e278e564970161a86e6652a63f58015bd2e7dc85de719e5bba6ef252","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"13d3f7c24d1951f7eccaeb591f09a7ad27b72a1f0d3c63a0a590154ec1d02b2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"59e0905d51abc3f260e60eaccaab70b634685836f78308d19447082523bd7935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"5dc7704bcbce0f4a7f9f81907bd6a202126d16c8173b589ea3e9a7efd501998e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8adea4c35fe3f9117bdf109bb92ed50cc3312cf016d9f42e8bb8d7d02a60e613","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8b45e8c93fa73392b7a5435ed12dc6c9c04d541483629a0f2c963d83febedd8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"11a1ea4698faced47725bf493b4aacc2124922e05d2aed8e74789e91de7a5717","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"60d8736178a39e3a5b4fef671fa2ef1330288eda4204ffbc60e31862e2471d95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"7d7a4bc924021f4fa04d801c41a3ac27180abe1576c70ed76bd677cebe7f2a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"a0af21048f0851309014ba5e3dc0d0f72ace62a5a58aa35d7e2b48b85ae30c13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113405158569283584","occurred_at":"2026-10-22T23:00:29Z","signature_id":"e51db32a50ceffa0ea6a335e06c32c375c606cf3ce2f73a2c6b002fa3305aea2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"bcbfe15488083c13d430775e87b2b3e5ab8941c2d5e7a7396b650e5580f81f7d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":16,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":16,"after_last_push_count":18,"trend_counts":[0,9,0,0,0,0,0,7],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34cbd3d68900\u003e: \n    failed to create HCP cluster ea-list8sj7pd: failed waiting for cluster=\"ea-list8sj7pd\" in resourcegroup=\"ea-list-qk95gg7gpcnb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n      \"name\": \"91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:32.731497539Z\",\n      \"endTime\": \"2026-10-28T20:43:34.905565303Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-collect-profiles, community-operators-catalog, openshift-apiserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, csi-snapshot-controller-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, kube-scheduler, ingress-operator, cluster-image-registry-operator, packageserver, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n            correlation ID: 92e26e96-cd60-4139-8146-f029c76dbd9e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34cbd3d68900\u003e: \n    failed to create HCP cluster ea-list8sj7pd: failed waiting for cluster=\"ea-list8sj7pd\" in resourcegroup=\"ea-list-qk95gg7gpcnb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n      \"name\": \"91a3c445-9466-4d25-8cd8-de90f532d7c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:32.731497539Z\",\n      \"endTime\": \"2026-10-28T20:43:34.905565303Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-collect-profiles, community-operators-catalog, openshift-apiserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, csi-snapshot-controller-operator, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, kube-scheduler, ingress-operator, cluster-image-registry-operator, packageserver, openshift-controller-manager, azure-cloud-controller-manager, cluster-network-operator, dns-operator, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/91a3c445-9466-4d25-8cd8-de90f532d7c6\n            correlation ID: 92e26e96-cd60-4139-8146-f029c76dbd9e\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x118c82e68960\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5fcqfhg6xm85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n      \"name\": \"241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:58:03.268943167Z\",\n      \"endTime\": \"2026-10-28T21:27:10.630246232Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, catalog-operator, dns-operator, olm-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-policy-controller, packageserver, machine-approver, openshift-controller-manager, cluster-storage-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n            correlation ID: 3553f468-7cef-45de-9639-6e7d0ba9a86e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x118c82e68960\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5fcqfhg6xm85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n      \"name\": \"241d3860-c6a5-496d-9a47-0ba2f3c27d4f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:58:03.268943167Z\",\n      \"endTime\": \"2026-10-28T21:27:10.630246232Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, catalog-operator, dns-operator, olm-operator, redhat-marketplace-catalog, csi-snapshot-controller-operator, cluster-image-registry-operator, konnectivity-agent, community-operators-catalog, control-plane-pki-operator, kube-controller-manager, cluster-policy-controller, packageserver, machine-approver, openshift-controller-manager, cluster-storage-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, ingress-operator, certified-operators-catalog, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/241d3860-c6a5-496d-9a47-0ba2f3c27d4f\n            correlation ID: 3553f468-7cef-45de-9639-6e7d0ba9a86e\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-21-mjsj484l4cg7/cluster-candidate-4-21-zqjdxv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e94b832e820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-zqjdxv: failed waiting for cluster=\"cluster-candidate-4-21-zqjdxv\" in resourcegroup=\"rg-candidate-4-21-mjsj484l4cg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n      \"name\": \"e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:06:06.856819198Z\",\n      \"endTime\": \"2026-10-28T20:35:15.000751357Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-storage-operator, community-operators-catalog, ingress-operator, olm-operator, cluster-network-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, olm-collect-profiles, openshift-route-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n            correlation ID: 0c3580d1-f96a-47b1-aaec-1d49dda7d990\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-21-mjsj484l4cg7/cluster-candidate-4-21-zqjdxv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3e94b832e820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-zqjdxv: failed waiting for cluster=\"cluster-candidate-4-21-zqjdxv\" in resourcegroup=\"rg-candidate-4-21-mjsj484l4cg7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n      \"name\": \"e7fb8359-1343-4d42-8975-4e247aae3bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:06:06.856819198Z\",\n      \"endTime\": \"2026-10-28T20:35:15.000751357Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, machine-approver, control-plane-pki-operator, openshift-apiserver, cluster-storage-operator, community-operators-catalog, ingress-operator, olm-operator, cluster-network-operator, packageserver, redhat-marketplace-catalog, cluster-version-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, olm-collect-profiles, openshift-route-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e7fb8359-1343-4d42-8975-4e247aae3bc5\n            correlation ID: 0c3580d1-f96a-47b1-aaec-1d49dda7d990\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"0d94360a0b18b44f41203aa87857d79b85bdb58471c8bb507781e2c8936eb1c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"36b9c2b22b1c0df002e2620d991914568e0046634273873e7ab53eaaf96416c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"b15ce3ecaa99dc5910d078a730555cb5295e82eb2dc3f11ddddbfa3af5dd845c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"b6ec2d0660375e02ba6f95200ef4f85ae3f323412ef6a27f3400068636542d7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3ce674c8fc052d08af13535dc5c626bc93710e8efd03a230ddab4a60c0946118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3e8f1658f5dc1d28d4efd52b26219407f6b0a170d15856d36bbedfd834ddbbe6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"f9bb6276ccece58dd6c1b6bfedcca5de455af5123fb87e6b0c63aeb1afa2b571","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"105a43b4bf54437760ddaec80d7274c76e0880fdd04198d3317c96b1e41e0747","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"179000d9fd7136a36d03dbd3bfaa3c8edbf92638e8ad95333390e6c1eadcd7dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"19248e56dbbe622fc0ab707175608011b12ef937b74bee06e900da1ff17799d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"7532d5417c94c768b19029dc4b5188f6d4f7b0525e268ed507fdd211a7691731","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"78bb0fd5dc8b2957b03be693a8b15a24b0624bf8afc1be3e13738a6518bfff22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"89edbfd547763da1be2ecb5ac3a37c254a734ffe88e2c8157270658bdc6cd116","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cb6c62fc5bd1672b019609253e5970cf0be960ae29f59adef3f3e3018a703317","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"e50814639bbed4fd159acc63cc76eb3b3ea44eedc3c0972c63ffedfeeb1d6b35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fa2dafb46bcd964c71b90a558a536dd7120ddbfa5212144072e39dc9ca2ed0c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1359d6bec8b8419e0698b5d08bade1b56f3dc616766a3a04ac2842544f5f224","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":16,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":16,"after_last_push_count":19,"trend_counts":[0,4,0,0,0,0,0,12],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-jshv8zgq8wk9/cluster-candidate-4-23-c6gbbp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xba6b52aa80\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-c6gbbp: failed waiting for cluster=\"cluster-candidate-4-23-c6gbbp\" in resourcegroup=\"rg-candidate-4-23-jshv8zgq8wk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n      \"name\": \"7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:10:20.02531103Z\",\n      \"endTime\": \"2026-10-28T20:39:25.022579343Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n            correlation ID: 15324628-570b-42b9-80a7-dbd435939211\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-23-jshv8zgq8wk9/cluster-candidate-4-23-c6gbbp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xba6b52aa80\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-c6gbbp: failed waiting for cluster=\"cluster-candidate-4-23-c6gbbp\" in resourcegroup=\"rg-candidate-4-23-jshv8zgq8wk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n      \"name\": \"7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:10:20.02531103Z\",\n      \"endTime\": \"2026-10-28T20:39:25.022579343Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7a88bbad-1a89-47a4-a4f2-a1b36a2086dd\n            correlation ID: 15324628-570b-42b9-80a7-dbd435939211\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-0-k2brb9cchnfb/cluster-candidate-5-0-c64hdj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x104ef3fa7e0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-c64hdj: failed waiting for cluster=\"cluster-candidate-5-0-c64hdj\" in resourcegroup=\"rg-candidate-5-0-k2brb9cchnfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n      \"name\": \"4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:10:48.049950576Z\",\n      \"endTime\": \"2026-10-28T20:39:55.034999436Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n            correlation ID: b9bdaba2-ec65-4fd2-abc1-4305bb14fac0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-5-0-k2brb9cchnfb/cluster-candidate-5-0-c64hdj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x104ef3fa7e0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-c64hdj: failed waiting for cluster=\"cluster-candidate-5-0-c64hdj\" in resourcegroup=\"rg-candidate-5-0-k2brb9cchnfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n      \"name\": \"4e3df969-37c8-42e0-bc9c-a513f08b762b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:10:48.049950576Z\",\n      \"endTime\": \"2026-10-28T20:39:55.034999436Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4e3df969-37c8-42e0-bc9c-a513f08b762b\n            correlation ID: b9bdaba2-ec65-4fd2-abc1-4305bb14fac0\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1da2d78427a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r4m67m749clh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n      \"name\": \"ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:26:05.081696809Z\",\n      \"endTime\": \"2026-10-28T19:55:13.864538233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n            correlation ID: 259a0ca4-d067-4227-b0b5-f5f22a7f7fe0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1da2d78427a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r4m67m749clh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n      \"name\": \"ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:26:05.081696809Z\",\n      \"endTime\": \"2026-10-28T19:55:13.864538233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ae56e62c-ca10-46a7-88ef-7fb6101ba7f0\n            correlation ID: 259a0ca4-d067-4227-b0b5-f5f22a7f7fe0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"1138e4fe25bc2fa499e07ce2095ef4b8e91229a94bc453bef3ca8e9f2be7f2bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"60d37d8dfdde6c0d2860f3361a3eb5f446365a64828f748da647a89a187b926b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"a8a2dc5232fbd8f841fd16941f0452da6dddb6c06224f8a02d8ea593a401d9d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"b89fb0e7cd2e1b39b36c5079ebdce4206fba71d4d1664e685853e2da6ae8985e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3f21188375c2a979f92a3e7d6b3f4e4da0853de8e5e02d474f98123639541247","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"542dcec1259b249fec86bbe3a02ee09976087e16aaa62ec1af2db79abd808885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"8476042b822d47e0f54de1129ee15563dd457884f1919c8e971a1c4b89973cad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"8661b303b20422ae894ac47a3a74293388ea0fd48db7bcc2b2dec9182a99758e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"87410719fd670b54f3aa86b66921cd3b52e5224e64794b89568b61f7af484cbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"9301f072fd6159abe387e800a4ec1b29d7b4df14329e35241bd16b50ac0a1265","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"b6ebb929d4286b9000cc4edc13f6a4fd80d5b6c19a793ad410789996849c7f23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"b945afde3acdd3156e44fe8bc3986c80cc24618929cebf7f823a1eb5b0391b63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"30b3256251eebbc6ad2f2d21745e8e3e6c242862e551efcc5662d3dab722db4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"37af00141384f00be88a064e4ee9337863ec4649e69343bc6f3df05230fa74a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"610533cebef7feda8d458dc8e967f9408265cfa9d327e5638860a65b8ee27823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"b542846af6df57de6bbea4872c890fa54411aeb1201b4a017c6a8bfb54ce2c21","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4mmnj.apps.aro.h4m7c3s6a2p4j7a.3hkp.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":12,"after_last_push_count":12,"also_in":["DEV","INT"],"trend_counts":[0,0,5,0,7,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-5gtk26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x187dce020660\u003e: \n    DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x187dce1941c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-5gtk26\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x187dce020660\u003e: \n    DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x187dce1941c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vvdwh.apps.aro.l5t3q5b8q6k0t1r.850o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33aec8dca6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33aec9a24040\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33aec8dca6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33aec9a24040\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bmd4l.apps.aro.v20261001.ez3w.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaa72ff3eb40\u003e: \n    DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xaa72fd7c9e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xaa72ff3eb40\u003e: \n    DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xaa72fd7c9e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7hxtd.apps.aro.v2i3p0q7g9o9o1k.cnfo.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"254849cbf5d8311752c99c9248a16fa04d7f6f2913e5fb83b7809a02a5a35916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"5eca3309ccd3e721e36f955090b7fbc956541d0a385657c2f9a6b84b63b4ed05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"63b9c0750fd00f7e0bfbdd70f47df6c7c8f5e5ec84bfe85b68318083b3ea2e5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"7d2b6c735bdffa857d31b502d0f673ff158743f511cc94c645b379eec32fe280","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"9d4340c3d5dd7a61b94734bf26b15b1bc36bf39a7633f96f82d30834ab5f4c8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"daa48f2da7fe25cf04788befd5b4892e4a30494e022bdb921224e7051f37fd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"e18d1395402ed17099a1459b75389ff956491b242d6bc7519d732c7e1751ed7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"0b5d6aa96dc387e24884d930781d5cd7d6557a061c28ab9733a3d3b1c2b4c58a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"dd7ef8065ee1042cfa5c3e7a73f09180f1300ed5a4e62e2f76b03643b5c75908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"1bd3c4c4ad3a9bdeb00d2c5694e282e664d811eb31be8ef577772f8ad9b54e05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"8e5db3d65c7b7608ee7e78aa4ee79fca7ba6f2e43840a7bfc37e8268baf454dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"aaeb63305e9f896148e97071cab84607802b6d3dc40093f2e77fe61a4284ba60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb9007365530c89d6f07bc37314eb0b69adbb1a18585a45e8a5c810a116009f2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":10,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["DEV"],"trend_counts":[0,3,0,0,0,0,0,7],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a9a9c302780\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dq6rz78qbzwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n      \"name\": \"36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:46.976794807Z\",\n      \"endTime\": \"2026-10-28T20:43:55.053369978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, ingress-operator, community-operators-catalog, dns-operator, packageserver, openshift-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n            correlation ID: 0302f0c3-0eb3-48dd-9510-05b3665bc29f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a9a9c302780\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dq6rz78qbzwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n      \"name\": \"36e9fb0c-0ea6-49be-917a-e6ad54933e8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:46.976794807Z\",\n      \"endTime\": \"2026-10-28T20:43:55.053369978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, ingress-operator, community-operators-catalog, dns-operator, packageserver, openshift-controller-manager, cluster-storage-operator, certified-operators-catalog, cluster-version-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, redhat-operators-catalog, csi-snapshot-controller-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36e9fb0c-0ea6-49be-917a-e6ad54933e8b\n            correlation ID: 0302f0c3-0eb3-48dd-9510-05b3665bc29f\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-p65ngr\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f72bfdcc60\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-p65ngr: failed waiting for cluster=\"cluster-zstream-4-22-p65ngr\" in resourcegroup=\"rg-zstream-upgrade-4-22-fk4qxqhpjj29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n      \"name\": \"8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:56.97734875Z\",\n      \"endTime\": \"2026-10-28T15:08:58.578863234Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, ingress-operator, csi-snapshot-controller-operator, community-operators-catalog, dns-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n            correlation ID: 094c53a6-4603-4b58-b443-8cca1dbc9dbb\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-p65ngr\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f72bfdcc60\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-p65ngr: failed waiting for cluster=\"cluster-zstream-4-22-p65ngr\" in resourcegroup=\"rg-zstream-upgrade-4-22-fk4qxqhpjj29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n      \"name\": \"8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:56.97734875Z\",\n      \"endTime\": \"2026-10-28T15:08:58.578863234Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-node-tuning-operator, ingress-operator, csi-snapshot-controller-operator, community-operators-catalog, dns-operator, cluster-image-registry-operator, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, cluster-network-operator, catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8ec9fe1f-0e5b-4c8f-94c5-92a6c0e3d366\n            correlation ID: 094c53a6-4603-4b58-b443-8cca1dbc9dbb\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32cee6f94b40\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gkhf8l89gfpt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n      \"name\": \"e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:19.663998842Z\",\n      \"endTime\": \"2026-10-28T15:08:28.451578554Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, redhat-operators-catalog, cluster-network-operator, openshift-route-controller-manager, dns-operator, packageserver, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, olm-operator, cluster-version-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n            correlation ID: 9764c911-c5b8-4802-ae2b-3bbd27636314\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32cee6f94b40\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gkhf8l89gfpt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n      \"name\": \"e910a1ec-64c8-4b3f-89a4-590c87d733cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:19.663998842Z\",\n      \"endTime\": \"2026-10-28T15:08:28.451578554Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, redhat-operators-catalog, cluster-network-operator, openshift-route-controller-manager, dns-operator, packageserver, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, ingress-operator, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, olm-operator, cluster-version-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e910a1ec-64c8-4b3f-89a4-590c87d733cf\n            correlation ID: 9764c911-c5b8-4802-ae2b-3bbd27636314\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"4b557052967c85109b6b45ef2f0fcb35fc9fc53f6bfbe6903a149cbafc57df96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"29f04c65df4c7248fa410a89d3babf8f77358c45cd2c52b5a6b3fc4d89ee052b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3b86e0ddfc2f3e17f4d42f49193fb43ee83d3b75659473c5f023382f9499d4b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"4f8d88d69095d88fc46b514f4dfd5f63a1f808b8d2d8db326a53b245ac6d1c10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"690d9a94c995b86ed981ba30241c5465df4455efe83abc764eb104e0bd36d163","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"97e9333a8653308cb25f8af8b95c6bd31e961edee95a0457bde72604aebc1ece","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"d5a5380e019074b97f540ce8686a93a598d62b7d02fc1170ba11db74040bc9fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a6864331b74f9520caaa01fb142d42bd934a1b337ffea6fcc51fd907a468b304","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d3dedfa37fc8f8f9d48c596be2117ea1ea85b98d4f8b0aba0451aa1f7bea81bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"fd02807286b5b83c10835eef9428f86b7fe1936bd74115749fc73fc5111727ba","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d469abb37fa12e6560dce7e8b62e834df7b39203aa112cb2dea653b5fb961b0a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":9,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":9,"after_last_push_count":15,"trend_counts":[0,2,0,0,0,0,0,7],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d67385a4720\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-292fs2ltfn6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n      \"name\": \"de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:18.787179838Z\",\n      \"endTime\": \"2026-10-28T19:53:23.770490111Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, kube-controller-manager, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, capi-provider, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, cluster-storage-operator, olm-collect-profiles, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, kube-scheduler, openshift-apiserver, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, dns-operator, olm-operator, openshift-controller-manager, packageserver, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n            correlation ID: 5f8d39c8-29ad-4bdc-8281-5d6497fadbe8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d67385a4720\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-292fs2ltfn6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n      \"name\": \"de6ffede-1c17-4a98-a65e-6e4b517b05bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:18.787179838Z\",\n      \"endTime\": \"2026-10-28T19:53:23.770490111Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, kube-controller-manager, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, capi-provider, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, community-operators-catalog, cluster-storage-operator, olm-collect-profiles, certified-operators-catalog, hosted-cluster-config-operator, cluster-image-registry-operator, kube-scheduler, openshift-apiserver, cluster-version-operator, openshift-route-controller-manager, azure-cloud-controller-manager, dns-operator, olm-operator, openshift-controller-manager, packageserver, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/de6ffede-1c17-4a98-a65e-6e4b517b05bf\n            correlation ID: 5f8d39c8-29ad-4bdc-8281-5d6497fadbe8\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a60cd6e680\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ggtvxjcb6gls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n      \"name\": \"6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:25.094565507Z\",\n      \"endTime\": \"2026-10-28T19:53:33.692195544Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-autoscaler, olm-collect-profiles, redhat-operators-catalog, kube-scheduler, ingress-operator, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, olm-operator, capi-provider, openshift-apiserver, csi-snapshot-controller-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, dns-operator, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, packageserver, openshift-route-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n            correlation ID: fb16cf1e-d070-4dc2-b0ad-1f7ce0d9da44\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a60cd6e680\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ggtvxjcb6gls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n      \"name\": \"6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:25.094565507Z\",\n      \"endTime\": \"2026-10-28T19:53:33.692195544Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-autoscaler, olm-collect-profiles, redhat-operators-catalog, kube-scheduler, ingress-operator, cluster-policy-controller, cluster-network-operator, cluster-image-registry-operator, olm-operator, capi-provider, openshift-apiserver, csi-snapshot-controller-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, dns-operator, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, packageserver, openshift-route-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, community-operators-catalog, machine-approver, control-plane-pki-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6b3c6bdf-43a8-4a44-8c7a-90f60ddcdcbc\n            correlation ID: fb16cf1e-d070-4dc2-b0ad-1f7ce0d9da44\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34142edd6a40\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-7s64tm5p2pnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\",\n      \"name\": \"f063aff6-fa6b-4397-9b14-f08322d800e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:14.723008935Z\",\n      \"endTime\": \"2026-10-28T19:53:23.809094912Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, capi-provider, cluster-version-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, olm-collect-profiles, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, packageserver, cluster-autoscaler, azure-cloud-controller-manager, olm-operator, redhat-marketplace-catalog, kube-controller-manager, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, cluster-policy-controller, community-operators-catalog, redhat-operators-catalog, control-plane-pki-operator, konnectivity-agent, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n            correlation ID: 9da90882-319b-4147-94e3-9897b59ff90d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:114]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34142edd6a40\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-7s64tm5p2pnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\",\n      \"name\": \"f063aff6-fa6b-4397-9b14-f08322d800e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:14.723008935Z\",\n      \"endTime\": \"2026-10-28T19:53:23.809094912Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, capi-provider, cluster-version-operator, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, olm-collect-profiles, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, packageserver, cluster-autoscaler, azure-cloud-controller-manager, olm-operator, redhat-marketplace-catalog, kube-controller-manager, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, cluster-policy-controller, community-operators-catalog, redhat-operators-catalog, control-plane-pki-operator, konnectivity-agent, certified-operators-catalog, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, machine-approver; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f063aff6-fa6b-4397-9b14-f08322d800e8\n            correlation ID: 9da90882-319b-4147-94e3-9897b59ff90d\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"50bccbab0fbf4c60d6c637d2f3d0712ba62570bb31026881eea6864443501319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"9ac7fd28791c8e7cfc0f7d973ab1a8ddc1ce8e355b2f754b95756b7b44066b8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"a19a06c858cf5713c455799ef09e3b0264f855a25c80378a222e92d8f7250eb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"b83842f62725a03eea143a87578ec93c1d0d4e416232fa1b48dba6d93d658a61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"194e6484baff053822ee2ec1a0e45066c34368d9f8361a9885d074f172876fe0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"29460d66c06c7032b48f41171cf575e427d2982eda0edd5c652064b13b4a3a96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"b02ac677d1d2b2e3d2754f6aaca8c25faaf2ac89e67671af0386ef8433265531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96c75f1ee40922482fa619e484710e1e93eb36cf6eaf5f78027744f7acdd09b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"d2d766d5aec5f079fb45688d592a83dedffde25333099326b9f7f5f8e100d8f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6fa414aeb81555490f7895124c219bd921c1327490d2fcd32eddc4d408170be","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":9,"after_last_push_count":13,"trend_counts":[0,3,0,0,0,0,0,6],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc983e72b20\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h8vnhnh28g7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\",\n      \"name\": \"338180fd-fd37-4118-8ce5-da19d007d2d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:20.13733405Z\",\n      \"endTime\": \"2026-10-28T19:53:23.880336461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-version-operator, kube-controller-manager, catalog-operator, openshift-controller-manager, dns-operator, olm-operator, cluster-network-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, konnectivity-agent, packageserver, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n            correlation ID: 92c6fcaf-99bb-49ef-8360-6f963a51cd80\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc983e72b20\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h8vnhnh28g7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\",\n      \"name\": \"338180fd-fd37-4118-8ce5-da19d007d2d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:20.13733405Z\",\n      \"endTime\": \"2026-10-28T19:53:23.880336461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, community-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, cluster-version-operator, kube-controller-manager, catalog-operator, openshift-controller-manager, dns-operator, olm-operator, cluster-network-operator, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, konnectivity-agent, packageserver, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/338180fd-fd37-4118-8ce5-da19d007d2d1\n            correlation ID: 92c6fcaf-99bb-49ef-8360-6f963a51cd80\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-swblz4\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3981bcd4a5c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-swblz4: failed waiting for cluster=\"cluster-zstream-4-22-swblz4\" in resourcegroup=\"rg-zstream-upgrade-4-22-nkrsx2q2bfk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\",\n      \"name\": \"fa758444-9da4-4261-b65a-fb9af0416aae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:11:05.048147318Z\",\n      \"endTime\": \"2026-10-28T20:40:15.036753709Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-policy-controller, kube-controller-manager, cluster-version-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, kube-scheduler, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, ingress-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n            correlation ID: 8fa02e62-e5c9-4192-8dc2-5346dc0423f9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-swblz4\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3981bcd4a5c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-swblz4: failed waiting for cluster=\"cluster-zstream-4-22-swblz4\" in resourcegroup=\"rg-zstream-upgrade-4-22-nkrsx2q2bfk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\",\n      \"name\": \"fa758444-9da4-4261-b65a-fb9af0416aae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:11:05.048147318Z\",\n      \"endTime\": \"2026-10-28T20:40:15.036753709Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-policy-controller, kube-controller-manager, cluster-version-operator, community-operators-catalog, dns-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, kube-scheduler, hosted-cluster-config-operator, cluster-image-registry-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, ingress-operator, packageserver, openshift-route-controller-manager, cluster-node-tuning-operator, cluster-storage-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fa758444-9da4-4261-b65a-fb9af0416aae\n            correlation ID: 8fa02e62-e5c9-4192-8dc2-5346dc0423f9\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-dn5ff6 with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9790da22600\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-dn5ff6: failed waiting for cluster=\"np-version-upgrade-cluster-dn5ff6\" in resourcegroup=\"rg-np-version-upgrade-dn5ff6-gc44wf2l4jv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n      \"name\": \"e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:55:25.009646036Z\",\n      \"endTime\": \"2026-10-28T21:24:30.625209621Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, ingress-operator, catalog-operator, kube-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n            correlation ID: 9f77dae1-d4df-43a9-aae8-bcfb3bc64031\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-dn5ff6 with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9790da22600\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-dn5ff6: failed waiting for cluster=\"np-version-upgrade-cluster-dn5ff6\" in resourcegroup=\"rg-np-version-upgrade-dn5ff6-gc44wf2l4jv6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n      \"name\": \"e487a831-bc1a-4554-9930-0ef5e5dd164e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:55:25.009646036Z\",\n      \"endTime\": \"2026-10-28T21:24:30.625209621Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, hosted-cluster-config-operator, dns-operator, ingress-operator, catalog-operator, kube-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, cluster-image-registry-operator, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, konnectivity-agent, olm-operator, redhat-marketplace-catalog, community-operators-catalog, certified-operators-catalog, cluster-version-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e487a831-bc1a-4554-9930-0ef5e5dd164e\n            correlation ID: 9f77dae1-d4df-43a9-aae8-bcfb3bc64031\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"2f48da34e6b9ed406b9605ee5e696280d567d993111a0dd4f10c0d136d538705","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"3240f3cd2cf3f8b8a71103086ee8514b0eaf1e7b232f5cde5bc18a32a344ae68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"4a6b63530c28e450b02962bbb0c52b15148d8644e14e86ec9e30122dd2845e15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"85a55c54ead7202802afe2a93f941819e12a2e1d54029fa4318f6226ee9536dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"9c6b7c5bde616122f7d939bcc3b19043ed6c57c2c81f1b37441d253d22839f33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"f2b41f64321f4d0c69f3e2ba802b7f90dfb2b447885bb1b355be09c939d2149d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"0e6ceee3cd3e93558b27b65ed689870c63b26dd3bc15ef1fe3541ac45579879e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"6d20fac6b2c664e06dd79b23d104b0707d6bcfcfba1dad3f3a5feeea9f41e97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"74f59f951658f16f8d54e8163c072eeb6209baea5d1369c72844e0d3da80a25d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c91c11f0a8c9c5ed9b313dc522cc8dc0e020935962a64510f3536da9ec41bb2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,1,0,0,0,0,0,6],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe23c97da540\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bsr8ks7wrjr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n      \"name\": \"3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:28:51.316103778Z\",\n      \"endTime\": \"2026-10-28T20:57:55.004654509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m35s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n            correlation ID: 5b00d586-2a8f-4b57-a9e1-84466b469fca\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe23c97da540\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-bsr8ks7wrjr6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n      \"name\": \"3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:28:51.316103778Z\",\n      \"endTime\": \"2026-10-28T20:57:55.004654509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m35s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3e534289-6bed-48ab-bc7c-f5aee1b8cdc3\n            correlation ID: 5b00d586-2a8f-4b57-a9e1-84466b469fca\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38a796b1a740\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lvjvqhcdtkrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\",\n      \"name\": \"499291aa-b279-429d-80ba-7786fc6cfa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:38:17.873619905Z\",\n      \"endTime\": \"2026-10-28T21:07:24.99192406Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n            correlation ID: 7577cb59-123a-459f-a8f3-f4a03306f15d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38a796b1a740\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lvjvqhcdtkrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\",\n      \"name\": \"499291aa-b279-429d-80ba-7786fc6cfa93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:38:17.873619905Z\",\n      \"endTime\": \"2026-10-28T21:07:24.99192406Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/499291aa-b279-429d-80ba-7786fc6cfa93\n            correlation ID: 7577cb59-123a-459f-a8f3-f4a03306f15d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cz8ptt with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c7b6c0a8440\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cz8ptt: failed waiting for cluster=\"np-version-upgrade-cluster-cz8ptt\" in resourcegroup=\"rg-np-version-upgrade-cz8ptt-hnxbd7dxv6dv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n      \"name\": \"6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:56:27.61784978Z\",\n      \"endTime\": \"2026-10-28T21:25:30.606619249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 25m13s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n            correlation ID: 4dbfaa07-bbd8-4943-933b-1b427a38701f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cz8ptt with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c7b6c0a8440\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cz8ptt: failed waiting for cluster=\"np-version-upgrade-cluster-cz8ptt\" in resourcegroup=\"rg-np-version-upgrade-cz8ptt-hnxbd7dxv6dv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n      \"name\": \"6a1a42fe-bb72-488b-adbf-55a400b476cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:56:27.61784978Z\",\n      \"endTime\": \"2026-10-28T21:25:30.606619249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 25m13s ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a1a42fe-bb72-488b-adbf-55a400b476cf\n            correlation ID: 4dbfaa07-bbd8-4943-933b-1b427a38701f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"1aa62d268d8e60f0844f153da0305ef176957779ea820d90f8034d253c93b02a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"5a4f21e5c2f30937e23a51b2fe564d63df2d5755f70f2cc3115bee5786c1490b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"a48400b513d98d37b9edb6f349d4f9b36b7a3f5371ab99314e6d5ace05ca1b4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ca8efaeb8e5ba4fae1aa48de2092e9d863408e6f54ac717a04257cf2b20d4e5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"7c952f7c533ba2ac25282224d93062045cb98f8a5d773ed2839acc1711d4e2eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"da5db98c965dd0440e1b301228d623d359946911d3815f2782f6cad5bd4513c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"9d48f25db5125a951413a47642b0efecb6d5d06c908e8b1e5fde16b1b20e1440","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"99c9ad984b7a477a15db113ef2ab36294a466de9394c6e9e134431a87132bed3","failure_pattern":"\u003cAzure error internals omitted\u003e","search_query":"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-lz7wggbwgh5c\" to finish creating: GET https://management.azure.com/subscription","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,7,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T14:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2a26cf54c078\u003e: \n    resource group \"np-autoscaling-ncn4v65w96fx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-ncn4v65w96fx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d43b55c4-7331-4381-b4fd-71f64a776433\n            correlation ID: 1d901265-7895-4b99-bf69-cdc848b0d0b5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1745c0262280\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddrtzn7r9n6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aa390de3-b57c-4004-ae00-8d256a07e3fb\n            correlation ID: 23755ba0-9dc6-42ab-bdd9-3208cee1312c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:175]: failed to get admin REST config for cluster np-version-upgrade-cluster-6klfkt\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31e04d7a70c0\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-6klfkt\" in resourcegroup=\"rg-np-version-upgrade-6klfkt-x72h4v22zqrn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b189d0f-2181-4aed-9e7f-51e3cb6c01ee\n            correlation ID: acc4317f-332d-41ad-92b6-3b3da8362c30\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"06ff6d227a4a567e0691243be5eccb0caeeaaa315bcc42dc0fea399f15c10736","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"2cc2591b74b05456e73089bc9b36c0f1f7eee60669fc135f3ce15bfec52d2937","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"4fa8b7fc464071338bdd4561210059a3e2c75c44318bf3c18a41e27d54116b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6f30c7a8231594aeb46f3d54216c8a117b477376ff72a9a27924ceba4f803e6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"aca4445f1d5148a20ae5983cba5de62c36b6c0a7b69db5f707a5e755bb5182a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"dd0b0b8df7b2166bc054ede8f1d58b688dc0dac527d6b2fd66ecb6ab9cb8ddc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fe77c534f2d11553fbb56bdbdd25fa10e6b63e20abc56954df520c44f8dde521","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3e116d57f9f7555e6eb9a73088d6f1c6ff99ff3cc60994fa14d15a5cebed5c04","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-ngqf2fwd7rsk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1fddbdc-ff80-4316-bce0-927ca36e9342","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,7,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T14:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x378ab2d1b4e0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-p9x242\" in resourcegroup=\"rg-np-skip-minor-p9x242-tqcdq45mhp7v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/773f88ac-95e8-4da7-bc09-57a818deb175\n            correlation ID: 8aa44ab4-58c2-423f-9a40-2f66225c45a7\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x8e881283980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8tlm65\" in resourcegroup=\"rg-np-noedge-8tlm65-6rtrf6fxs8pp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/657d92b1-0420-465c-a20f-051929954d68\n            correlation ID: da25ea79-7ee4-4aa5-a3cd-24d74907d5d3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11adfdef64a0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nm2jdjsx4m9t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7ab31f32-e5a9-4966-9e78-a32d1786c989\n            correlation ID: f811a401-7f9a-42bd-861f-59e24ffc103f\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"022548c08c7f3d00dcc2c6ccfbb90c7621484b0f6cc92b83282d239b2bf97fec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"0705e7ce0972714a7b11f80b2c8aff85f61cdc6febe2637f5a146fe039b1f537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"3fb0e1297078af590b2d19465e2eb4a0832651168d99f0176c33e483a935dcbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6d820660123e1f2b46bada2acc37a3d7025031ef13070c0759df5cdf0c7b2de4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"b8acb5f0bafdb5a3735bb27828a6b4f3c5003359238914449eaa893ab955598b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"e4a1953e3872ab73515a0438ce007de9e2def00ae922c3087992e0fce1014e4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"fd25ada7f7c17331da353bb415e7abc6bfe24733b76e6dae5911dba3aa007638","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18ae850b518e4bd2d9611948378626f037231a583300d80b18fe149e69964b54","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[0,2,0,0,0,0,0,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14a5e2008300\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k46j9j85xq5z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\",\n      \"name\": \"a961a70a-5b5a-4175-bb58-82fb36beb466\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:26:12.743400806Z\",\n      \"endTime\": \"2026-10-28T19:55:13.7057976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n            correlation ID: 5835c446-bdab-409a-8175-cbf5109c1943\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14a5e2008300\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k46j9j85xq5z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\",\n      \"name\": \"a961a70a-5b5a-4175-bb58-82fb36beb466\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:26:12.743400806Z\",\n      \"endTime\": \"2026-10-28T19:55:13.7057976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a961a70a-5b5a-4175-bb58-82fb36beb466\n            correlation ID: 5835c446-bdab-409a-8175-cbf5109c1943\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1683f5097100\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mhvkjzh45hwc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n      \"name\": \"e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:28:18.023653922Z\",\n      \"endTime\": \"2026-10-28T19:57:23.841118908Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n            correlation ID: 7d36723c-bee1-4adc-86af-798e4647be9f\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1683f5097100\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mhvkjzh45hwc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n      \"name\": \"e9a36944-a3a3-4eaf-9fdc-98e8262c979b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:28:18.023653922Z\",\n      \"endTime\": \"2026-10-28T19:57:23.841118908Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e9a36944-a3a3-4eaf-9fdc-98e8262c979b\n            correlation ID: 7d36723c-bee1-4adc-86af-798e4647be9f\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-jqd7cs\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13016dc5e5e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-jqd7cs: failed waiting for cluster=\"cp-ystream-upgrade-5-0-jqd7cs\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ft5czs88jtdf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n      \"name\": \"1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:34.851456324Z\",\n      \"endTime\": \"2026-10-28T20:41:34.873934441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n            correlation ID: 03cba8d0-c85c-4970-a348-de9b3f376ff4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-jqd7cs\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13016dc5e5e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-jqd7cs: failed waiting for cluster=\"cp-ystream-upgrade-5-0-jqd7cs\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-ft5czs88jtdf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n      \"name\": \"1dc97088-f5bb-4932-bd4c-14b5c83e689d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:12:34.851456324Z\",\n      \"endTime\": \"2026-10-28T20:41:34.873934441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dc97088-f5bb-4932-bd4c-14b5c83e689d\n            correlation ID: 03cba8d0-c85c-4970-a348-de9b3f376ff4\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"1c7eaaca2159b4db61db1cbd5d59ac643c49c639ef01458521ea8976229989c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"26c36401fd91e0588393f9c87cccfc2015e7a28fc80e00ea485b3b3fa225f6ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ef331e26d1592f4da8d5a5d7392e5167b1dc464a19ba6a9519786efd70bd586d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"2f0c8223166367ebb908c5e6dac5e21ad27bbd34413df694da368440c482b45c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"10bb0bdb748cd2f51cfdd05065854a330965cf2734479264377206588e187b5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"8181e1d728fd2f9d65a27b1a6f385b120f5a9c516932134b128d158f80fe37e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"987e1cacd319bfda34a108a0d83818ae1469e71ef6b784405ebc07582fe31b0c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":6,"after_last_push_count":10,"trend_counts":[0,2,0,0,0,0,0,4],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c03b21d6c20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7fv9wkcmzdx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n      \"name\": \"7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:37.658468995Z\",\n      \"endTime\": \"2026-10-28T20:43:44.89685741Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n            correlation ID: 1b096272-5f83-459d-a9b3-80242ffba49d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c03b21d6c20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7fv9wkcmzdx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n      \"name\": \"7f5aec2c-eaa8-4eef-ae1b-398990595f26\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:37.658468995Z\",\n      \"endTime\": \"2026-10-28T20:43:44.89685741Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7f5aec2c-eaa8-4eef-ae1b-398990595f26\n            correlation ID: 1b096272-5f83-459d-a9b3-80242ffba49d\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t67p9t with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbe0fc68500\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t67p9t: failed waiting for cluster=\"np-version-upgrade-cluster-t67p9t\" in resourcegroup=\"rg-np-version-upgrade-t67p9t-kpsvm25jmqsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\",\n      \"name\": \"9e35dee8-3994-4ba7-9234-74a03d742751\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:15.512939467Z\",\n      \"endTime\": \"2026-10-28T15:50:17.639783703Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n            correlation ID: 8d785f53-f7f0-40a1-aa91-751064d7e050\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t67p9t with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbe0fc68500\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t67p9t: failed waiting for cluster=\"np-version-upgrade-cluster-t67p9t\" in resourcegroup=\"rg-np-version-upgrade-t67p9t-kpsvm25jmqsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\",\n      \"name\": \"9e35dee8-3994-4ba7-9234-74a03d742751\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:15.512939467Z\",\n      \"endTime\": \"2026-10-28T15:50:17.639783703Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9e35dee8-3994-4ba7-9234-74a03d742751\n            correlation ID: 8d785f53-f7f0-40a1-aa91-751064d7e050\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24f1c7fcefa0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bhc75ljrxzvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\",\n      \"name\": \"3ee91092-37d4-4687-8899-086b94094407\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:42:47.010012275Z\",\n      \"endTime\": \"2026-10-28T14:11:50.916418144Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n            correlation ID: 881a23bb-07d5-4161-bd1d-8f0790ff0f30\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24f1c7fcefa0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bhc75ljrxzvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\",\n      \"name\": \"3ee91092-37d4-4687-8899-086b94094407\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:42:47.010012275Z\",\n      \"endTime\": \"2026-10-28T14:11:50.916418144Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ee91092-37d4-4687-8899-086b94094407\n            correlation ID: 881a23bb-07d5-4161-bd1d-8f0790ff0f30\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"71c974d1a4355b686678eaeec937bb1114fad8167d13110b0bd5deddaafdb12a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"5bb392afcae0a94809f7d0c4935691ce0cb5e3f427769bf81d509c7da81d71c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"8a964c6e955aa1074acee9f587db3b0f87b653b49549c6bf100107183dfd73cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"964c8299f8415fec5f061be2c91ce28301f9952572870c4aee2bce32e9c6371e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"222ffc9c563b0b4467dc853c34c77c4694cb2ce41c4c51c01cfbc0965f89f195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3918b5053e2c1b6926508abb7cd5f09e8c960484f7c759d88c7b0673e703991d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":6,"after_last_push_count":6,"also_in":["DEV","INT"],"trend_counts":[0,0,6,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T23:38:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n    DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n    DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n    DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n    DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"a23174899568b6c480cde29c16f107e2df7fb592b2576c4598c4dd3b3ed64763","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"d9fbf23376585cdfaf80950f7a3f5e8696d3b0c22688297d50b6821493fe3062","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"eeff509555b3bd3bab3c3359799b2c2e597d7c59acb0e88f24bfbf61670e37e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"fffc6a8a7c348c164995910dd1c108865b80e864e9a58f247d4be169e8a399d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"476aa5c24e557ad6a577878e82a38cad972775741e8c95bde27dcf8780a1adf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"baa9afa3bcbcd0c44804b70a02b3154a77e55643457f2cbdcc513051780298c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["INT"],"trend_counts":[0,0,3,0,2,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x33a1f1315158\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x33a1f0cea780\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x33a1f0bd2f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x33a1f13e4360\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0x33a1f1315158\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x33a1f0cea780\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x33a1f0bd2f00\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x33a1f13e4360\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-59cgj.apps.aro.s4i2d1i2e0s3o3m.o0jt.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xeeb7f5e0a68\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xeeb7f3b69e0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xeeb7f5137a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xeeb7f512040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xeeb7f5e0a68\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xeeb7f3b69e0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xeeb7f5137a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xeeb7f512040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4kcwc.apps.aro.cilium-cl.en3z.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"0ac27c2c2596a79cd4371dcfd3d01134184bca542ebdf146502aaa026ae80229","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"78fd5d6ed20f79e2553486e2f00d359fec4b08aeeed6379beabf7bf312ae9a76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"05baccf373c01bd6f7562dcbc3236b5780751cf9ec7eae7616fd70c9c8c3da1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"103697c5a8db15b41fc2a08d46ab15e59d6e4653e09a4e1f747b198e64c261d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"959d7136fa8fa3534927661a3d60094a7592b6d059d009358f71e15228fe76e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":4,"after_last_push_count":9,"trend_counts":[0,0,1,2,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-19"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-25T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x5c10dd80030\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x5c10d35ddc0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10d35dda0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10db35540\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x5c10dd88020\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10dd88000\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10dbae320\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x5c10dd80030\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x5c10d35ddc0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10d35dda0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10db35540\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x5c10dd88020\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x5c10dd88000\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x5c10dbae320\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114492215458795520","occurred_at":"2026-10-25T23:00:04Z","signature_id":"ead8c420db783411710bfdc18c3ac594eab0616d6007cfdac94e624578b57afc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"4a296b8184626c9cf0da738c096b9a7efd69214488b2d8d7c86178f33312d094","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"f4dbd9fc7f76d3d491864efc557c3c22f1a4295a0221ba7f6fc875ded0a8dd16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"489c87a4b234518c7dfe6b37a3d19c5c6af589cc404fb24f4b84bb4855abdbd3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,0,3],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xfa629f24ea0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zkznqc7xhzd4\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-zkznqc7xhzd4\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xfa629c70d20\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xfa629f24ea0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zkznqc7xhzd4\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-zkznqc7xhzd4\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xfa629c70d20\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zkznqc7xhzd4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1f0baa544120\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1f0baa40b610\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88\",\n            },\n            \u003c*fmt.wrapErrors | 0x1f0baa5440c0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0x1f0baa40b610\u003e0x1f0baa40b610 (cyclic reference),\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1f0baa544120\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-n4hrqndpjj88\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1f0baa40b610\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88\",\n            },\n            \u003c*fmt.wrapErrors | 0x1f0baa5440c0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-n4hrqndpjj88\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-n4hrqndpjj88, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0x1f0baa40b610\u003e0x1f0baa40b610 (cyclic reference),\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x22e10818450\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-992c2j7mq2x8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-992c2j7mq2x8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x22e1036b8e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x22e10818450\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-992c2j7mq2x8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-992c2j7mq2x8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x22e1036b8e0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-992c2j7mq2x8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"46ada0aa48fab5345ad452546fa116726a981cc504e0cec6c061ca9182ae0b45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"8c96c1bd310c60c6a4f69d6210e10ae7e4099a5d02be37a4168d23f20c67b47c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"a4eb8e9210102fc87c30ce72625cc97530479383ab1754550319fd1e35820302","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"f4c91ed9f592165df1c3c27e2fd22c4b93dc93ae321a35275a1f80ee00060e71","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,1,0,0,0,0,0,2],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-2lr49","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","INT"],"trend_counts":[0,0,2,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d8dba328160\u003e: \n    DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d8dba3280e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d8dba328160\u003e: \n    DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d8dba3280e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-52rjm.apps.aro.n5x4o1n9j0c5w9n.rqrz.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"076cacfa7561e03f7335026846eac17d1438353985ba609f1c98efdf0e3de954","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"68fc0f1ea6cf0f44536f6ef401fc1e1215329535859894d52835b6c6b59f5866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"16e5dc89de6f908080887b19103bb8d46e6a6d30176df898a38a625fe63e5380","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ef96fcb3cedbbf63865b36d0e47bdd70f7edc69989eb675136e8b13a4ecb570d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[0,0,0,0,1,0,2,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x341fcf187480\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6d2jd2rhkhpr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\",\n      \"name\": \"354ac5e7-358b-4ba7-b33a-800a398e3d90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:18:08.702878641Z\",\n      \"endTime\": \"2026-10-27T21:37:14.197553955Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\n            correlation ID: 2d032843-08dc-4bd5-a784-b1e25c32c35a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x341fcf187480\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6d2jd2rhkhpr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\",\n      \"name\": \"354ac5e7-358b-4ba7-b33a-800a398e3d90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T21:18:08.702878641Z\",\n      \"endTime\": \"2026-10-27T21:37:14.197553955Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/354ac5e7-358b-4ba7-b33a-800a398e3d90\n            correlation ID: 2d032843-08dc-4bd5-a784-b1e25c32c35a\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2462c6c0b540\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bl9lknth6jps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\",\n      \"name\": \"97c439db-9ae6-4e86-85bd-71d21a1fe1e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T18:48:30.242794404Z\",\n      \"endTime\": \"2026-10-27T19:07:38.465503369Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\n            correlation ID: f85455a5-fb5a-4cdd-8e2d-d003c7fb917e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2462c6c0b540\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bl9lknth6jps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\",\n      \"name\": \"97c439db-9ae6-4e86-85bd-71d21a1fe1e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T18:48:30.242794404Z\",\n      \"endTime\": \"2026-10-27T19:07:38.465503369Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/97c439db-9ae6-4e86-85bd-71d21a1fe1e7\n            correlation ID: f85455a5-fb5a-4cdd-8e2d-d003c7fb917e\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a837c45580\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m8kxkqwbf5w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"name\": \"64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T09:39:04.403310399Z\",\n      \"endTime\": \"2026-10-25T09:58:04.042996716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n            correlation ID: a329644d-c403-4d74-b2b5-fcc949c9882c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a837c45580\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-m8kxkqwbf5w7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"name\": \"64b6150d-1b43-49ba-9a27-51fef85bb940\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T09:39:04.403310399Z\",\n      \"endTime\": \"2026-10-25T09:58:04.042996716Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/64b6150d-1b43-49ba-9a27-51fef85bb940\n            correlation ID: a329644d-c403-4d74-b2b5-fcc949c9882c\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115176579041595392","occurred_at":"2026-10-27T20:19:29Z","signature_id":"d4bc68183bce520581535b610de7dd8a8c531f314d19ecf8feb5c372ad243b69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115132439494922240","occurred_at":"2026-10-27T17:13:43Z","signature_id":"b5a167c4947d0a94153f5f027cb4d6157c127dc8f203d4dfd862e0f7f9c21472","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"eb4c9f692ebf3a9ebdd322c6273f89d4ee6b231c71d079f6aa6d3f3c0db70247","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[0,2,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T04:22:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x144b1f0220a8\u003e: \n    resource group \"ingress-cert-le-pqlv4rswg88h\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pqlv4rswg88h\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n      \"name\": \"f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:05:17.896484451Z\",\n      \"endTime\": \"2026-10-28T14:29:22.39155595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tnqroht6vs8i90m1kekfo1uicveo895 still exists (deletion dispatched at 2026-10-28T14:05:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n            correlation ID: 3972a876-9ac5-4cc1-a07c-3501cf7d75bd\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x144b1f0220a8\u003e: \n    resource group \"ingress-cert-le-pqlv4rswg88h\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pqlv4rswg88h\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n      \"name\": \"f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:05:17.896484451Z\",\n      \"endTime\": \"2026-10-28T14:29:22.39155595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tnqroht6vs8i90m1kekfo1uicveo895 still exists (deletion dispatched at 2026-10-28T14:05:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f2c1c705-5b8d-4bbe-98a3-39f7905fcad2\n            correlation ID: 3972a876-9ac5-4cc1-a07c-3501cf7d75bd\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3eb0083dc060\u003e: \n    resource group \"private-keyvault-dtwhsct926n6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dtwhsct926n6\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"name\": \"baf15df3-25ce-45bb-9f9e-ce6a52637752\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:08:07.811493592Z\",\n      \"endTime\": \"2026-10-22T05:32:16.130563503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjkddd25goskik0nf1lqaqp6mgu12qg still exists (deletion dispatched at 2026-10-22T05:08:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/baf15df3-25ce-45bb-9f9e-ce6a52637752\n            correlation ID: a461bf17-668e-42e4-a6f7-c14803c5d886\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x16e186f694b8\u003e: \n    resource group \"ephemeral-osdisk-k7jcdd2kqwvh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k7jcdd2kqwvh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"name\": \"36a51701-9d7a-4ad4-860f-d0b45481d5d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:23:14.821436082Z\",\n      \"endTime\": \"2026-10-22T03:47:22.914912373Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjj0cikkek0bc3m05t20hehmct394q5 still exists (deletion dispatched at 2026-10-22T03:23:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36a51701-9d7a-4ad4-860f-d0b45481d5d2\n            correlation ID: b94afee3-9d6b-4695-a5b0-3f0b5bdf76c0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"a7a82dbb2248dec4c46a4ed0be66a247d60fbf4440afa724a4ffcf4486ebe06a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"8a9ded4e992ead494e672f2c33210492690aab0b14e2de539e32782deff72275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"cbeccff849e6332c34aba59d393fdbae4f3ec1001f41174ac4f54440f6912fdd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4c37c8f822619cf7691a6f98235190ed213a6e85dc73231a2f532590e39f61c8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api, ignition-server]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,2,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x320c1cc145a0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z8bmdl4f2djz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n      \"name\": \"6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:43:08.173393078Z\",\n      \"endTime\": \"2026-10-28T14:12:10.809270873Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n            correlation ID: 446aa419-d2ee-4b05-829a-2ef1a0102b8b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x320c1cc145a0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-z8bmdl4f2djz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n      \"name\": \"6f2afbda-3f8a-4abe-93da-0b42da0b54b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:43:08.173393078Z\",\n      \"endTime\": \"2026-10-28T14:12:10.809270873Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2afbda-3f8a-4abe-93da-0b42da0b54b6\n            correlation ID: 446aa419-d2ee-4b05-829a-2ef1a0102b8b\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bbjx5c\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2052152a2720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-bbjx5c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bbjx5c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-56g2lp75qrkf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"name\": \"02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:23:39.532402776Z\",\n      \"endTime\": \"2026-10-22T02:52:40.523543888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/02ef6437-1d48-43c3-ad2d-b3b8f18f1b53\n            correlation ID: 43512322-23b0-4759-b738-201cec5eb6f3\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-gmftj87c69s6/cluster-candidate-5-0-25snrj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ed3b41bc940\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-25snrj: failed waiting for cluster=\"cluster-candidate-5-0-25snrj\" in resourcegroup=\"rg-candidate-5-0-gmftj87c69s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"name\": \"27425e0a-5b12-47d6-9369-3a3a90aa3e86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:48.683878735Z\",\n      \"endTime\": \"2026-10-22T02:50:50.395326627Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 2 unavailable replicas, ignition-server deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/27425e0a-5b12-47d6-9369-3a3a90aa3e86\n            correlation ID: 52162704-0f0b-4a93-a71e-0f363b9222b6\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"3bfe96f1806920438e26c083d745013e6c6dc2bf1dacb1307401c4f7751ba7d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"43ca0ef5137db4c92687ca44af2e5269c14f5a14f38084e6f18c59c868e18577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"66fe9486ebbdab2264fdca019d93b069038bbe27a27a787cb2a4da95a08e65ec","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c52db8e6d8927ba84d619660a76292f72a9f97d6689c92dc4710e6d83404ab89","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-20T04:23:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16967f5def20\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7wppdvdd55v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\",\n      \"name\": \"54478daa-47e5-4530-91c2-342cc29a1749\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:21.628677591Z\",\n      \"endTime\": \"2026-10-28T19:53:23.838328082Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.20 is Partial (want Completed), started 26m9s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n            correlation ID: 7c6978d3-1a22-47a9-861d-b27f5d6a3f88\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x16967f5def20\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-z7wppdvdd55v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\",\n      \"name\": \"54478daa-47e5-4530-91c2-342cc29a1749\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:21.628677591Z\",\n      \"endTime\": \"2026-10-28T19:53:23.838328082Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.20 is Partial (want Completed), started 26m9s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/54478daa-47e5-4530-91c2-342cc29a1749\n            correlation ID: 7c6978d3-1a22-47a9-861d-b27f5d6a3f88\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d00e53ba4e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mpx2klxjj7k9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n      \"name\": \"3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:17.131948207Z\",\n      \"endTime\": \"2026-10-28T19:53:23.815769706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n            correlation ID: c1be684a-2dcb-4ac2-af67-f7bd12ce1f70\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d00e53ba4e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mpx2klxjj7k9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n      \"name\": \"3d9941f3-6ece-4bcd-980b-9e776d1e3227\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:24:17.131948207Z\",\n      \"endTime\": \"2026-10-28T19:53:23.815769706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3d9941f3-6ece-4bcd-980b-9e776d1e3227\n            correlation ID: c1be684a-2dcb-4ac2-af67-f7bd12ce1f70\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-95f5bw with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c6ab1a4a4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-95f5bw: failed waiting for cluster=\"np-version-upgrade-cluster-95f5bw\" in resourcegroup=\"rg-np-version-upgrade-95f5bw-mppp9bg5hxs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\",\n      \"name\": \"13f26b26-2324-48a4-82df-d8b324b13996\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:52:21.783931077Z\",\n      \"endTime\": \"2026-10-28T21:21:30.723052246Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 26m5s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n            correlation ID: 9f0bfbd0-f67d-4e88-9467-75878a8fe604\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-95f5bw with version 4.21.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c6ab1a4a4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-95f5bw: failed waiting for cluster=\"np-version-upgrade-cluster-95f5bw\" in resourcegroup=\"rg-np-version-upgrade-95f5bw-mppp9bg5hxs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\",\n      \"name\": \"13f26b26-2324-48a4-82df-d8b324b13996\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:52:21.783931077Z\",\n      \"endTime\": \"2026-10-28T21:21:30.723052246Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 26m5s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f26b26-2324-48a4-82df-d8b324b13996\n            correlation ID: 9f0bfbd0-f67d-4e88-9467-75878a8fe604\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"4f9202c02773536be9847760ee3cf57fdb731b7b1a80187ad25cf10ef4a097f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"7bf017f71b8954e81509c7a98e71ca2c7f3c8fb5fa6883390b6c8cb356446a5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"b6dabb2ae36ec121af2506af67029f1dd28ff29ddcc71f7d20abbd9a72a3223a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e49174cb063fa817fdf1f0104686d88cbe5af47e82f4a43d41af0318ce04c3fb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd2fa34b2580\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-8rnw5kdb6p2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\",\n      \"name\": \"e56720ec-9cdb-4896-bffe-b107388330fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:26.535304826Z\",\n      \"endTime\": \"2026-10-28T15:50:27.739063015Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, cluster-image-registry-operator, cluster-autoscaler, hosted-cluster-config-operator, cluster-network-operator, dns-operator, catalog-operator, community-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, ingress-operator, kube-controller-manager, cluster-storage-operator, konnectivity-agent, packageserver, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, olm-collect-profiles, csi-snapshot-controller-operator, redhat-operators-catalog, olm-operator, redhat-marketplace-catalog, machine-approver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n            correlation ID: 87508886-196b-41dd-b14a-e423d6c69bea\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd2fa34b2580\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-8rnw5kdb6p2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\",\n      \"name\": \"e56720ec-9cdb-4896-bffe-b107388330fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:26.535304826Z\",\n      \"endTime\": \"2026-10-28T15:50:27.739063015Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, cluster-policy-controller, cluster-image-registry-operator, cluster-autoscaler, hosted-cluster-config-operator, cluster-network-operator, dns-operator, catalog-operator, community-operators-catalog, control-plane-pki-operator, azure-cloud-controller-manager, ingress-operator, kube-controller-manager, cluster-storage-operator, konnectivity-agent, packageserver, kube-scheduler, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator, olm-collect-profiles, csi-snapshot-controller-operator, redhat-operators-catalog, olm-operator, redhat-marketplace-catalog, machine-approver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e56720ec-9cdb-4896-bffe-b107388330fa\n            correlation ID: 87508886-196b-41dd-b14a-e423d6c69bea\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x283eb71121c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wqltj9p8d5tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"name\": \"07e1caae-294a-4885-b57d-e584ce7036c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:11:14.830364623Z\",\n      \"endTime\": \"2026-10-22T03:40:15.05749374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, kube-controller-manager, kube-scheduler, openshift-apiserver, cluster-storage-operator, ingress-operator, community-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, cluster-image-registry-operator, konnectivity-agent, machine-approver, csi-snapshot-controller-operator, packageserver, cluster-policy-controller, dns-operator, catalog-operator, redhat-operators-catalog, cluster-autoscaler, olm-operator, control-plane-pki-operator, hosted-cluster-config-operator, openshift-controller-manager, cluster-node-tuning-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07e1caae-294a-4885-b57d-e584ce7036c1\n            correlation ID: 042010a3-104c-478c-8958-4456058069f0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"50872cc62b458472eb61bba5961cd6a3739cefd259772bc8671550bbb691403b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"ac79520c264a03cdae111d50487f33410c268ee3536a309598ebaf8fa5495141","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ea5f0567a445fa592fabdccbbe6d57a8abba6455be492502fa1afb3dba38a3f5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33d30568cba0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rn9mbdwr669k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\",\n      \"name\": \"30b1eeed-8a58-491f-a679-903582bd2428\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:06.381277659Z\",\n      \"endTime\": \"2026-10-28T15:50:07.62996819Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, kube-scheduler, dns-operator, certified-operators-catalog, openshift-controller-manager, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, machine-approver, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, cluster-autoscaler, control-plane-pki-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n            correlation ID: 375547a6-70a5-4d64-8ca8-71de42a27606\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33d30568cba0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rn9mbdwr669k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\",\n      \"name\": \"30b1eeed-8a58-491f-a679-903582bd2428\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:06.381277659Z\",\n      \"endTime\": \"2026-10-28T15:50:07.62996819Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator, cluster-policy-controller, olm-operator, kube-scheduler, dns-operator, certified-operators-catalog, openshift-controller-manager, cluster-image-registry-operator, olm-collect-profiles, redhat-operators-catalog, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, konnectivity-agent, machine-approver, kube-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, cluster-autoscaler, control-plane-pki-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/30b1eeed-8a58-491f-a679-903582bd2428\n            correlation ID: 375547a6-70a5-4d64-8ca8-71de42a27606\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-d44bvbtv22rz/cluster-candidate-4-23-f2rpqw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x126c5cc0e60\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-f2rpqw: failed waiting for cluster=\"cluster-candidate-4-23-f2rpqw\" in resourcegroup=\"rg-candidate-4-23-d44bvbtv22rz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"name\": \"efe747a0-4a74-4970-aae9-c5a7e67a1f5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.82450299Z\",\n      \"endTime\": \"2026-10-22T02:50:40.503091649Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, catalog-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, konnectivity-agent, olm-operator, certified-operators-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, kube-storage-version-migrator, community-operators-catalog, openshift-apiserver, cluster-policy-controller, cluster-autoscaler, cluster-network-operator, dns-operator, control-plane-pki-operator, kube-scheduler, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, ingress-operator, cluster-image-registry-operator, olm-collect-profiles; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/efe747a0-4a74-4970-aae9-c5a7e67a1f5b\n            correlation ID: 08db8465-69b5-4f99-aabf-f55441aa0ace\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"7ba55d85c0848e7e7fe424c701434aa2745b7d5574cad1964a7875eac6fadb52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"5a8c81cfc5de57e2ac5cb1f344b5d4c790956481a2f66094e81c030894d26e74","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0d6ef0f747748f3897783008b2ad9c103a4591a86a275b0ca77bdd29c34e6b0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12961691c220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjbhtbfkbk4z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n      \"name\": \"b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T02:05:53.148166937Z\",\n      \"endTime\": \"2026-10-28T02:34:59.674611282Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n            correlation ID: 5a9d9ef9-c6e1-45c7-a60d-afc7b67a11c1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12961691c220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjbhtbfkbk4z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n      \"name\": \"b9e8b50e-a67a-443d-9059-6dace5a0e1b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T02:05:53.148166937Z\",\n      \"endTime\": \"2026-10-28T02:34:59.674611282Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9e8b50e-a67a-443d-9059-6dace5a0e1b3\n            correlation ID: 5a9d9ef9-c6e1-45c7-a60d-afc7b67a11c1\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1138be74e660\u003e: \n    failed to create HCP cluster np-dg-minor-gsd8rp: failed waiting for cluster=\"np-dg-minor-gsd8rp\" in resourcegroup=\"rg-np-dg-minor-gsd8rp-mv6pwhz6p5qt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\",\n      \"name\": \"b44a2fb9-01ac-4953-9693-2e6baa16d65c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T17:26:50.386134218Z\",\n      \"endTime\": \"2026-10-26T17:55:54.373104457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\n            correlation ID: f252e869-38a8-48ec-8d30-9db8cb2d096a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1138be74e660\u003e: \n    failed to create HCP cluster np-dg-minor-gsd8rp: failed waiting for cluster=\"np-dg-minor-gsd8rp\" in resourcegroup=\"rg-np-dg-minor-gsd8rp-mv6pwhz6p5qt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\",\n      \"name\": \"b44a2fb9-01ac-4953-9693-2e6baa16d65c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T17:26:50.386134218Z\",\n      \"endTime\": \"2026-10-26T17:55:54.373104457Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; [tlsCertificates] TLS certificates not yet confirmed; [clusterServiceClusterStatus] cluster service is pending; [defaultIngressServingCertificate] default ingress certificate not yet observed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b44a2fb9-01ac-4953-9693-2e6baa16d65c\n            correlation ID: f252e869-38a8-48ec-8d30-9db8cb2d096a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2115262480262369280","occurred_at":"2026-10-28T02:00:49Z","signature_id":"f37db33d97ba0fa58d50483c591fa7b5d1cded288f97755f3eda96c7fd078fd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114749140742180864","occurred_at":"2026-10-26T16:00:27Z","signature_id":"6c097dbc55dc852dcd42b6dac9b1fd76eefe2547adf5185002575ae5dee30591","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12","2026-10-19"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-23T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2f7afefabce0\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2f7afec3aa20\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2f7afec3aa00\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2f7aff732b40\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2f7afee8d220\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2f7afee8d200\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2f7afee1fae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2f7afefabce0\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2f7afec3aa20\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2f7afec3aa00\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2f7aff732b40\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2f7afee8d220\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2f7afee8d200\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2f7afee1fae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"583bb04d0678cc06b27764a775ef03a5d7ac139cfbdebcf4fb7aaea2e364c2d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"e854c2baa0b762928bd1753c83c278e1be7a0f0d62fdeaf4af4e9edab0cc8f65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ebb8d128ac36a20abe9223f2c5925135d7f381d45a13394b4f2869aab4d48252","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV","STG"],"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-m28flb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x27a8bf192000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x27a8c0a02f80\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-m28flb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x27a8bf192000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x27a8c0a02f80\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n            observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n            observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114854607891992576","occurred_at":"2026-10-26T23:00:05Z","signature_id":"303177dd1f87db48a25c9ca11072eaaaaff18c0dcc22516704d88d73534c2648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","signature_id":"79dfce6cb10e96a30452837453f16449a11349eb353010336138903c85b23a84","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x471790ed8f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x4717a706200\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x471790ed8f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x4717a706200\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x397e13438ac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x397e134eabf0\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x397e13438ac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*errors.errorString | 0x397e134eabf0\u003e{\n                s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"928b1de9065772d3f2c7e201fb9ae101d2fb2df8e5b2cbd4ff7e82df961c73ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"bda22742bb4bcfe82726d83de21fb4e400993a95b279453234202a917cefb495","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3ce2ba89b8e9686083a83dadac7501168f4efe9a7eacff619946e50e9db1c762","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-25T08:24:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x1bbed2a5bb90\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x1bbed27f1680\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x1bbed27f1450\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x1bbed2a5bb90\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x1bbed27f1680\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x1bbed27f1450\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.zjlv.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n    {\n        Op: \"Get\",\n        URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n        Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n            Op: \"dial\",\n            Net: \"tcp\",\n            Source: nil,\n            Addr: nil,\n            Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n                UnwrapErr: nil,\n                Err: \"no such host\",\n                Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n                Server: \"172.30.0.10:53\",\n                IsTimeout: false,\n                IsTemporary: false,\n                IsNotFound: true,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2114271800467656704","occurred_at":"2026-10-25T08:24:13Z","signature_id":"2e48272f7c6962293e37e27585cc8a5c9247db1c6e77cd891e766919f7108086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"86a169f78b189ca410e85207ae5eedf317f888d81a7fbf1f8ae05a6c40c5927e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cabce77ed6ae1cccb39d04a144cecb9c14bb38197c32bd8205cedc9a33963b08","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-21-76q9tvzg8rn9/cluster-candidate-4-21-5lg29k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x143589e84400\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5lg29k: failed waiting for cluster=\"cluster-candidate-4-21-5lg29k\" in resourcegroup=\"rg-candidate-4-21-76q9tvzg8rn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"name\": \"edb6a2fa-979a-4af5-a995-cd3897780e01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:35.877670552Z\",\n      \"endTime\": \"2026-10-22T02:50:40.514192951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/edb6a2fa-979a-4af5-a995-cd3897780e01\n            correlation ID: 85af8528-9942-40bd-9e4c-05e55063f9c4\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-gp6dgjrbtz82/cluster-candidate-4-22-qlh9hc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1de55d1398a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-qlh9hc: failed waiting for cluster=\"cluster-candidate-4-22-qlh9hc\" in resourcegroup=\"rg-candidate-4-22-gp6dgjrbtz82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"name\": \"8cc15b2a-8360-41a2-8a2a-f0798be9df0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:21:54.045486495Z\",\n      \"endTime\": \"2026-10-22T02:51:00.49750165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cc15b2a-8360-41a2-8a2a-f0798be9df0d\n            correlation ID: 1eeaec8e-068a-41c6-8cee-4c44e2977d7a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"3ec3f52024a46a1f942ae3af19740a4993c313ed0bf4cec5b820ab716098ba7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"96e526637370066fb982529359f8fb2c1357e11506ca6d9024d12db800540150","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b286b88e4cdf2beb25fd0ed17c671477745b97698c63efca626d18c2b4af0e4","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T20:07:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n    \u003c[]string | len:2, cap:2\u003e: [\n        \"console (Available=False)\",\n        \"ingress (Available=False)\",\n    ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n    \u003c[]string | len:2, cap:2\u003e: [\n        \"console (Available=False)\",\n        \"ingress (Available=False)\",\n    ]\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"3bfc2a89887bfd392938b2d517fdbf090bf267eaca9f9ccad980b9907856ef39","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"169f70dba2351443907226ac5ccc1485fb8e510b011ab14b2d8af60b14150235","failure_pattern":"DNS for route host agnhost-e2e-sample-app-lpqxs","search_query":"DNS for route host agnhost-e2e-sample-app-lpqxs","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T20:07:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n    DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n    DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","signature_id":"cdfd5eadc24e0be0c150b2244b8232f0a89d3c7047aecd3074d3a83bba715ea6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"bf28464fdc326495b2d6ca9e56df879b27c4e19498ca8f2c4dfd708b81c66524","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-s9k5d8\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x219ad233aca0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-s9k5d8: failed waiting for cluster=\"cluster-zstream-5-0-s9k5d8\" in resourcegroup=\"rg-zstream-upgrade-5-0-n6xzg4dztfw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\",\n      \"name\": \"e768a2f1-bf75-4afe-9acf-741914b73091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:11:31.970768227Z\",\n      \"endTime\": \"2026-10-28T20:40:35.054714998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, kube-controller-manager, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, machine-approver, kube-scheduler, openshift-apiserver, openshift-controller-manager, cluster-image-registry-operator, catalog-operator, capi-provider, kube-storage-version-migrator, olm-operator, packageserver, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n            correlation ID: 12e6213e-1673-4bde-bd5f-964565fbe8d0\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-s9k5d8\" with version 5.0.0-rc.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x219ad233aca0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-s9k5d8: failed waiting for cluster=\"cluster-zstream-5-0-s9k5d8\" in resourcegroup=\"rg-zstream-upgrade-5-0-n6xzg4dztfw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\",\n      \"name\": \"e768a2f1-bf75-4afe-9acf-741914b73091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:11:31.970768227Z\",\n      \"endTime\": \"2026-10-28T20:40:35.054714998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, azure-cloud-controller-manager, csi-snapshot-controller-operator, kube-controller-manager, cluster-version-operator, cluster-policy-controller, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, cluster-autoscaler, machine-approver, kube-scheduler, openshift-apiserver, openshift-controller-manager, cluster-image-registry-operator, catalog-operator, capi-provider, kube-storage-version-migrator, olm-operator, packageserver, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e768a2f1-bf75-4afe-9acf-741914b73091\n            correlation ID: 12e6213e-1673-4bde-bd5f-964565fbe8d0\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"83a55ef34f01654f6dc3600d12e4c084f0f9f1b138bd67071de6c9fc2819ebea","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1edd5970e96b87d9128b141b52d9d5df9ec69d99da03a6f49cf9571a2753ec7e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c9f11d305c0\u003e: \n    failed to create HCP cluster np-downgrade-pgblzc: failed waiting for cluster=\"np-downgrade-pgblzc\" in resourcegroup=\"rg-np-downgrade-pgblzc-w296czt7c4h5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n      \"name\": \"c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:57:10.002650956Z\",\n      \"endTime\": \"2026-10-28T21:26:10.63929862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, cluster-version-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, kube-scheduler, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, dns-operator, cluster-image-registry-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n            correlation ID: 3652d2cb-5ad7-44b4-9342-18162d59e0da\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c9f11d305c0\u003e: \n    failed to create HCP cluster np-downgrade-pgblzc: failed waiting for cluster=\"np-downgrade-pgblzc\" in resourcegroup=\"rg-np-downgrade-pgblzc-w296czt7c4h5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n      \"name\": \"c3f52426-50e3-477e-b6d2-730dd7ef4644\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:57:10.002650956Z\",\n      \"endTime\": \"2026-10-28T21:26:10.63929862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, cluster-version-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, kube-scheduler, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, community-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, openshift-route-controller-manager, dns-operator, cluster-image-registry-operator, olm-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3f52426-50e3-477e-b6d2-730dd7ef4644\n            correlation ID: 3652d2cb-5ad7-44b4-9342-18162d59e0da\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"78ae657176cafaf184588b18ae514d272d0816a3ee3c0b87eea4e62207620774","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"37985da5f97bff5443a20ce7eddabdbfd55a0839dc6b22fe1cac002e46ccec2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-controller-manager, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3da3f19209c0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-p9cvh6b6lj55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n      \"name\": \"9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:36.875741431Z\",\n      \"endTime\": \"2026-10-28T20:43:44.887394978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, dns-operator, catalog-operator, certified-operators-catalog, csi-snapshot-controller-operator, redhat-marketplace-catalog, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, packageserver, ingress-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n            correlation ID: 84abeaa9-4051-418b-b066-32246401955a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:98]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3da3f19209c0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-p9cvh6b6lj55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n      \"name\": \"9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:14:36.875741431Z\",\n      \"endTime\": \"2026-10-28T20:43:44.887394978Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, dns-operator, catalog-operator, certified-operators-catalog, csi-snapshot-controller-operator, redhat-marketplace-catalog, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, packageserver, ingress-operator, cluster-image-registry-operator, kube-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c5a9e43-a50b-4146-b8d2-de67c8eeeb10\n            correlation ID: 84abeaa9-4051-418b-b066-32246401955a\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"19b82bfd691cff869c6c199782be4eee2c38c25cbdf6f1433c85a73266a7223c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2be81406fae3edbe2084aa4ec6e8c4bdd917e7ac49e9527c271da2794674dd42","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2tf7pf\nUnexpected error:\n    \u003c*fmt.wrapError | 0x257516f84460\u003e: \n    failed to create HCP cluster np-noedge-2tf7pf: failed waiting for cluster=\"np-noedge-2tf7pf\" in resourcegroup=\"rg-np-noedge-2tf7pf-xb8pwtjzh82j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\",\n      \"name\": \"92128bbf-b05b-4a42-b590-261b4b996ffc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:31.746407548Z\",\n      \"endTime\": \"2026-10-28T15:50:37.764425706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, community-operators-catalog, ingress-operator, csi-snapshot-controller-operator, packageserver, cluster-image-registry-operator, dns-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n            correlation ID: b2029c0e-c3d1-46d0-9bac-f3b40a92dc09\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2tf7pf\nUnexpected error:\n    \u003c*fmt.wrapError | 0x257516f84460\u003e: \n    failed to create HCP cluster np-noedge-2tf7pf: failed waiting for cluster=\"np-noedge-2tf7pf\" in resourcegroup=\"rg-np-noedge-2tf7pf-xb8pwtjzh82j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\",\n      \"name\": \"92128bbf-b05b-4a42-b590-261b4b996ffc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:31.746407548Z\",\n      \"endTime\": \"2026-10-28T15:50:37.764425706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-controller-manager, openshift-route-controller-manager, redhat-operators-catalog, community-operators-catalog, ingress-operator, csi-snapshot-controller-operator, packageserver, cluster-image-registry-operator, dns-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-storage-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/92128bbf-b05b-4a42-b590-261b4b996ffc\n            correlation ID: b2029c0e-c3d1-46d0-9bac-f3b40a92dc09\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"caa3b54dcae0c5626ec06f2239837c5a5dd7fe48e4b5279bc29225d20116ebaa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"04fa2dff8b6b51d28d362de7ef46f5028d2496b5105a9c36a86aac17ab5e65e0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2840a3490800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9tjwn77pblt6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n      \"name\": \"f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:48:29.562853635Z\",\n      \"endTime\": \"2026-10-28T21:17:30.601011159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, konnectivity-agent, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n            correlation ID: 12f04724-ca79-49ae-ac94-baef82ad2b08\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2840a3490800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9tjwn77pblt6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n      \"name\": \"f493db72-5a08-4ef3-8c73-4d055bd2007e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:48:29.562853635Z\",\n      \"endTime\": \"2026-10-28T21:17:30.601011159Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-route-controller-manager, cluster-policy-controller, konnectivity-agent, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f493db72-5a08-4ef3-8c73-4d055bd2007e\n            correlation ID: 12f04724-ca79-49ae-ac94-baef82ad2b08\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"e374f7b8196855ef661f3e1f5e456bd341e5c59f7cf586f6925e971d4f0eceaa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b95030e680b85c36e813048d300579274516d1f1a72079e03a11261579e993f4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, kube-scheduler, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3290e061a680\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qdsgxxj79bfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n      \"name\": \"5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:57:27.436526224Z\",\n      \"endTime\": \"2026-10-28T21:26:30.622804028Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, cluster-policy-controller, openshift-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n            correlation ID: 2fd33249-a131-4a13-a1a2-03c755186865\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3290e061a680\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-qdsgxxj79bfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n      \"name\": \"5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:57:27.436526224Z\",\n      \"endTime\": \"2026-10-28T21:26:30.622804028Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, cluster-policy-controller, openshift-controller-manager, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e4ef4af-8f03-4c8b-bbbb-d82bdd2ed0e9\n            correlation ID: 2fd33249-a131-4a13-a1a2-03c755186865\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"9fef6cdd46bcccbc965ee603bb9552f0ff376c7d7b5cc6240f49957e233a60ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0377bb07338fe0f98769c18440cfba81bee2c9117babcbbc37fcf8db2825921a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, openshift-controller-manager, openshift-route-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d947782ea20\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w5rx25h5j8dw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n      \"name\": \"0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:01:08.393167298Z\",\n      \"endTime\": \"2026-10-28T20:30:16.146752142Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n            correlation ID: 1eb4abcf-932d-4da4-828c-22172d8177f5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:168]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d947782ea20\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-w5rx25h5j8dw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n      \"name\": \"0a9f3022-f355-43e2-9259-81dfd53aa74e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:01:08.393167298Z\",\n      \"endTime\": \"2026-10-28T20:30:16.146752142Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0a9f3022-f355-43e2-9259-81dfd53aa74e\n            correlation ID: 1eb4abcf-932d-4da4-828c-22172d8177f5\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"ce6bdc393948e6a40a02a08779ba758155c6f242cf794d5aedb733749470627f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1865bb63dfef83736260561a6034bb4b023ad00e9cb60922026ce37a8215cdee","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, kube-scheduler, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xgw4k6 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f2f6cd22960\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-xgw4k6: failed waiting for cluster=\"np-version-upgrade-cluster-xgw4k6\" in resourcegroup=\"rg-np-version-upgrade-xgw4k6-fb2pqq2pp9pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\",\n      \"name\": \"67af219e-4e40-4a49-ae77-1d482039a909\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:37.478622401Z\",\n      \"endTime\": \"2026-10-28T15:50:37.723667825Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, kube-scheduler, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n            correlation ID: b05be546-d520-46a0-8713-ceb437dfacc8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xgw4k6 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f2f6cd22960\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-xgw4k6: failed waiting for cluster=\"np-version-upgrade-cluster-xgw4k6\" in resourcegroup=\"rg-np-version-upgrade-xgw4k6-fb2pqq2pp9pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\",\n      \"name\": \"67af219e-4e40-4a49-ae77-1d482039a909\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:37.478622401Z\",\n      \"endTime\": \"2026-10-28T15:50:37.723667825Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, kube-scheduler, cluster-policy-controller, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/67af219e-4e40-4a49-ae77-1d482039a909\n            correlation ID: b05be546-d520-46a0-8713-ceb437dfacc8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"f0d67bf88f0b1e061ef2f75989731d2852c725ccd7d7653a9aedb9b57213d67e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2a4dc40bdb155c7b5e7f0e2213ef4c3de0123c4f0e14b53fa40355751f06e680","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28ad82ebac20\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-8dptc8vk8gtv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n      \"name\": \"d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:01:11.67931549Z\",\n      \"endTime\": \"2026-10-28T20:30:16.178438369Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n            correlation ID: 66a835d6-6997-4516-8fbc-7fd19b5ac311\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28ad82ebac20\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-8dptc8vk8gtv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n      \"name\": \"d2708466-dcce-4112-be7d-6e0e6a3a5117\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:01:11.67931549Z\",\n      \"endTime\": \"2026-10-28T20:30:16.178438369Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d2708466-dcce-4112-be7d-6e0e6a3a5117\n            correlation ID: 66a835d6-6997-4516-8fbc-7fd19b5ac311\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"1b8fcec76c5e298c087b6dd13ca411d382423c18b384cd3f5c84450ce6644b1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18e6120f83ea7fcc84253d55c25e1ca8355971e7accf1adbd2fc6e3196e9463a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d893d926940\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9zwtss6f6vt9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n      \"name\": \"7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:20.447678223Z\",\n      \"endTime\": \"2026-10-28T15:08:28.688541915Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n            correlation ID: 2b84f196-091a-41b2-98d7-60b50644dab4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:91]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d893d926940\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9zwtss6f6vt9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n      \"name\": \"7bc6a3bc-3918-4c91-853b-58acb06d5c9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T14:39:20.447678223Z\",\n      \"endTime\": \"2026-10-28T15:08:28.688541915Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, openshift-route-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7bc6a3bc-3918-4c91-853b-58acb06d5c9a\n            correlation ID: 2b84f196-091a-41b2-98d7-60b50644dab4\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"f6d47638df5c178dc4841bfd12e3a2a7720138c28c5031cd942b4f59a9c78972","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cbf83efdfcbd15587be949946e10b03eab123e752a2d391c330556621de8f835","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent, kube-controller-manager, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186bf96476a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f9vz9brpqmbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n      \"name\": \"07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:42:56.996297525Z\",\n      \"endTime\": \"2026-10-28T14:12:00.914226401Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n            correlation ID: b4a6856e-5332-498a-8060-89b9ebb7175d\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186bf96476a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f9vz9brpqmbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n      \"name\": \"07dce0e9-a762-440b-9ccd-61b15fa63bf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T13:42:56.996297525Z\",\n      \"endTime\": \"2026-10-28T14:12:00.914226401Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07dce0e9-a762-440b-9ccd-61b15fa63bf8\n            correlation ID: b4a6856e-5332-498a-8060-89b9ebb7175d\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"234ea90fae8d4c8e76a6d720fc3746417508af7ee1d3235413e8209300f96430","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fffa36f85bbbff9b9e35ac04e5e1995d6605346e9b86d9872df10486fda0e14f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T01:09:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ede8eb28c00\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qzb5xz65mghw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"name\": \"43a33130-1505-4488-8d2c-b0f13de47cff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T03:04:08.329961738Z\",\n      \"endTime\": \"2026-10-22T03:33:15.140670523Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-route-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/43a33130-1505-4488-8d2c-b0f13de47cff\n            correlation ID: 2c680bb1-7f2a-45c1-a484-bbb38ed78677\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113075208892977152","occurred_at":"2026-10-22T01:09:23Z","signature_id":"a7827d5ad5ffd2899a170b78f82132d6020ab7d4fd7644e60ac3a32d23a3e66e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9114d91c0b1a3cf2341a472ef6ddec9850e63302a0009ffa178fc2dbceb63d9f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1308b97a65a0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-79mst7jcpg45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n      \"name\": \"94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:58:09.127282806Z\",\n      \"endTime\": \"2026-10-28T21:27:10.682711777Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n            correlation ID: 49b88924-064f-4f52-9480-109cf607a390\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1308b97a65a0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-79mst7jcpg45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n      \"name\": \"94a5e742-d4dc-4cd5-bf1f-345949f27c96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:58:09.127282806Z\",\n      \"endTime\": \"2026-10-28T21:27:10.682711777Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.e0m1j3e6v6o1i5i.b0k7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/94a5e742-d4dc-4cd5-bf1f-345949f27c96\n            correlation ID: 49b88924-064f-4f52-9480-109cf607a390\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"af33a2dad0cf09fb287a7b4234fdf373fb47cb8419d2c52ae3c2a78d006a72e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9edc10b3275c8fa4fef403327c8f3071abe3dac4d93e21bb75e84a9f8c3924d6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7c22833c460\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kn67v2kn8ztj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n      \"name\": \"d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:06.28860999Z\",\n      \"endTime\": \"2026-10-28T15:50:07.63308505Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n            correlation ID: de1e5d6e-772f-44e9-aec7-7b1dbc7151a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7c22833c460\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kn67v2kn8ztj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n      \"name\": \"d1ad5413-0977-4c3c-abd1-937a8f55bfc2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T15:21:06.28860999Z\",\n      \"endTime\": \"2026-10-28T15:50:07.63308505Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io failed: Get \\\"https://api-int.p8k0e2b5k4j2e4n.nvz7.eastus2euap.aroapp-hcp.io:443/healthz?verbose\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1ad5413-0977-4c3c-abd1-937a8f55bfc2\n            correlation ID: de1e5d6e-772f-44e9-aec7-7b1dbc7151a7\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115430933703168000","occurred_at":"2026-10-28T13:09:17Z","signature_id":"c6fc8da7a6c3fdef78901982a9e72825daf9ba1be7cf33cceedbd6a810b64ca4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"59fd7347675f085770dffc68f405a63ce20be2172042c801154cf178794d1b7b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-20T04:23:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2df91d94ebc0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c9fcp5td85qq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n      \"name\": \"57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:52:14.429975606Z\",\n      \"endTime\": \"2026-10-28T20:21:16.149275515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n            correlation ID: 7836a1e8-ab2a-417c-9954-360298a60ae2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2df91d94ebc0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c9fcp5td85qq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n      \"name\": \"57f0f2bf-9284-42db-8d6e-b13db18de25a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T19:52:14.429975606Z\",\n      \"endTime\": \"2026-10-28T20:21:16.149275515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.40 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/57f0f2bf-9284-42db-8d6e-b13db18de25a\n            correlation ID: 7836a1e8-ab2a-417c-9954-360298a60ae2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"86b26fa2d9dd9e5aa9c08e0aee435bdda933b68e0c00ce13c0df48be507bb165","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"facbcb79129e6159c9b5455a0c79d2a7fec07d572d42676eaad216f2de41ba83","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g5tsjc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd7eab0d0620\u003e: \n    failed to create HCP cluster np-noedge-g5tsjc: failed waiting for cluster=\"np-noedge-g5tsjc\" in resourcegroup=\"rg-np-noedge-g5tsjc-pf5lpmzmdnbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\",\n      \"name\": \"36728157-067b-4375-a959-9c9ce43e534b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:56:26.269135944Z\",\n      \"endTime\": \"2026-10-28T21:25:30.687402107Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m57s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n            correlation ID: ff24c2ee-7aff-4c46-91cc-8fde998760f5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g5tsjc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd7eab0d0620\u003e: \n    failed to create HCP cluster np-noedge-g5tsjc: failed waiting for cluster=\"np-noedge-g5tsjc\" in resourcegroup=\"rg-np-noedge-g5tsjc-pf5lpmzmdnbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\",\n      \"name\": \"36728157-067b-4375-a959-9c9ce43e534b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T20:56:26.269135944Z\",\n      \"endTime\": \"2026-10-28T21:25:30.687402107Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.44 is Partial (want Completed), started 24m57s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/36728157-067b-4375-a959-9c9ce43e534b\n            correlation ID: ff24c2ee-7aff-4c46-91cc-8fde998760f5\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115519015970410496","occurred_at":"2026-10-28T19:00:12Z","signature_id":"fd12dbccec9d53cfb9ca9f8ad5c7dd7ccc1665086dac2a9a1dfaa7d925d023d0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ef53b575bef4613518d7c9b1d1c586f30d8987486df16b669a252c7b7cdd2e07","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: interfaces creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27665d95f6c0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-dhqxbzp84k6j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\",\n      \"name\": \"7b5c9bdf-eb40-4dfb-a637-a90c825e7658\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T03:14:20.369455888Z\",\n      \"endTime\": \"2026-10-27T03:33:20.394901036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-gqrvp-269v2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine ephemeral-disk-ephemeral-np-gqrvp-q67fk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-gqrvp-269v2: Creating: interfaces creating or updating\\nMachine ephemeral-disk-ephemeral-np-gqrvp-q67fk: Creating: interfaces creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\n            correlation ID: a243a81d-f70c-497e-9a6c-d0f5669ed0d7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27665d95f6c0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-dhqxbzp84k6j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\",\n      \"name\": \"7b5c9bdf-eb40-4dfb-a637-a90c825e7658\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T03:14:20.369455888Z\",\n      \"endTime\": \"2026-10-27T03:33:20.394901036Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-gqrvp-269v2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine ephemeral-disk-ephemeral-np-gqrvp-q67fk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-gqrvp-269v2: Creating: interfaces creating or updating\\nMachine ephemeral-disk-ephemeral-np-gqrvp-q67fk: Creating: interfaces creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7b5c9bdf-eb40-4dfb-a637-a90c825e7658\n            correlation ID: a243a81d-f70c-497e-9a6c-d0f5669ed0d7\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2114899906912063488","occurred_at":"2026-10-27T02:00:05Z","signature_id":"997b9ff275d4628102f111804c4a3921713fb1706985c143fe0ad5af317b98a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":4,"also_in":["DEV","INT"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-23T14:34:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n      \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n            correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n      \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n            correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113640155058278400","occurred_at":"2026-10-23T14:34:17Z","signature_id":"83894b496f209ee26cbf0e4d1f1a178c6a6d6f9bdf0c14187a9b22ad01f74877","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":5,"also_in":["DEV"],"trend_counts":[1,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-21T16:01:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x161cee87c740\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9tq8tw8pkh5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\",\n      \"name\": \"aa729d08-abec-481d-a5d5-3a517e315df9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T18:45:35.525302675Z\",\n      \"endTime\": \"2026-10-27T19:04:37.944268372Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9tq8tw8pkh5c--managed/providers/Microsoft.Compute/virtualMachines/q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9tq8tw8pkh5c--managed/providers/Microsoft.Compute/virtualMachines/q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\n            correlation ID: bbfa9866-a276-4fbe-b94b-bc8519188ea5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x161cee87c740\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9tq8tw8pkh5c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\",\n      \"name\": \"aa729d08-abec-481d-a5d5-3a517e315df9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T18:45:35.525302675Z\",\n      \"endTime\": \"2026-10-27T19:04:37.944268372Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9tq8tw8pkh5c--managed/providers/Microsoft.Compute/virtualMachines/q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-9tq8tw8pkh5c--managed/providers/Microsoft.Compute/virtualMachines/q5i4q4e4c0d2d0d-arm64-vm-np-1-qfjp4-9wwh2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/aa729d08-abec-481d-a5d5-3a517e315df9\n            correlation ID: bbfa9866-a276-4fbe-b94b-bc8519188ea5\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2115142680974462976","occurred_at":"2026-10-27T17:28:45Z","signature_id":"7aa547873bec896935c66a761f18ad01967f373df0172fa9711de2d70adaca08","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6fe02653aee74c57d72dfff05e4f572539da57d10c19d6d05ba4d0781d80c774","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T11:31:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n      \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n            correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n      \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n            correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113594207997267968","occurred_at":"2026-10-23T11:31:42Z","signature_id":"a4e598ab2e945986cd1ad69b1bb3c65fbdf94c05e97f34879cec8e6778f23b99","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b9ac8e13134319c4cdeffb9d46f43716baeb67c5239b74632e6bf7675546e7af","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T04:22:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5070b80fa0\u003e: \n    failed to create NodePool np-lp5s2l: failed waiting for nodepool=\"np-lp5s2l\" for cluster \"cluster-candidate-5-0-lp5s2l\" in resourcegroup=\"rg-candidate-5-0-m8vcgvcqk4pg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"name\": \"fd2644a3-f725-418e-af3f-2985a0768c42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T05:20:50.362804863Z\",\n      \"endTime\": \"2026-10-22T05:39:55.652634149Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u1j3l3a4s2x8k8u-np-lp5s2l-lng7p-xl2t7: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/fd2644a3-f725-418e-af3f-2985a0768c42\n            correlation ID: b2009fac-4672-460e-84fc-49211ba0f615\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113123849489879040","occurred_at":"2026-10-22T04:22:40Z","signature_id":"bace6221e58a5d21dfe413ecbb1909743fd07f6b8c34c992d650b4df4119cf4a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","STG"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-24T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","signature_id":"ddcef7f1372fb43fa0a1e2ed4da6ae9b117c2451538d5508f48d3045f3f05809","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ec625aa187990188920942c47358d3eb1c7eda37bdf66a6d43c4b3578a43526","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-23T23:38:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x2d202c117968\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x2d202c117968\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","signature_id":"9dfcee82299c274c6cbfd06b3f9fb3c7b36ad00e636501927c19be2e530212db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"12318b0632fb674fc784fdd096e34ba234440b0cb9593f4ff4c36061f31941fe","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" in cluster=\"\u003ccluster\u003e\" resourcegroup=\"\u003cresource-group\u003e\" to finish updating: GET \u003curl\u003e","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-22T14:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32f3fb206560\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qk6fnz9248pm\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE UNAVAILABLE\n    --------------------------------------------------------------------------------\n    Response contained no body\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8a81c298-fb2e-4c06-ab5b-f047a74ffe2b\n            correlation ID: bd21a41c-3537-483b-a104-ebc5b06bde9b\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113274341746544640","occurred_at":"2026-10-22T14:20:40Z","signature_id":"6b805215ae5be9a20646349cb83be2fc300cf869ef7d92e96198f6d3b0f5e8fe","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":24,"failed_runs":16,"raw_occurrences":69,"matched_occurrences":67,"runs_affected":14},"rows":[{"environment":"stg","failure_pattern_id":"e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":24,"runs_affected":6,"failed_runs":6,"run_impact_percent":25,"anchor_occurrences":24,"after_last_push_count":28,"also_in":["DEV","PROD"],"trend_counts":[0,5,9,5,0,2,3,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-12","2026-10-19"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"5f7305d0b3b91c61eb8e4f56e572507a452aa85cc00ec307edf3b8e9bfddebb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"70755a49efebe5ef08cdb2d0691a83edd6fc97c800ce0e052ac99acb90b8d7f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"e6435ae321fc270025d85586172e832697f963663729776cb49988ed3cb99186","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"c3805a868ccf40727d210350cbe8cf8e87890bfe820ceae256095c60d80a32e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"f974db37c7eb3d9bbc97893012fdea83ddca24ef2f512221011480246752904a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"0bb1d0ea133617e80d73304b99ac404df807e39474511338a75212981c65b43f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"0e1afae7cbd1abf142582a196bedbea42edec5611d2065f7c4bb033cd593be75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"2ffe87b37e421112fe72ccfc08f02a1a91c7cfef672785163cac957acb46ccd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"72f41d6f8493cfe1096ed81660653df35a55af60dee4a19bc7c25df92ed036af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"cd265acb51f92274cce95dbd49a1905edba77c027e3c409a14b12da380e56fb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"49bc2dba2bfaf378bf9a212deaa8b7172aaf792a0742a6cf81eadfafcfc7f9c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"4e3ce0964a5b3bd20469f8350751ac1287a305e1677cf080184b7f571285c485","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"68f626a4183079baec214426eb915954fa39eaded38cbb3491a96faab9b91433","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"bb1a13221a54012b00614d6c3045119dc40aa91f975a154b83fbf8702349ed00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"edf85baa7b056a0719d25c42570bd84e02fed466b6414d44af1029ab25659f96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"209b1990a8a2f436458cc8cb9df5e8c94119b64beeae3da3339e7634dd11fc08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"8d3f197b2c580940d4d0cf0708fb900f557fc2023b09444c9fee41bc6a5642aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"9edf2fd2a048ea5fd699b522030694ea0094920a5d1e251b878b9adf70f3d5f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"f07d7dccb44b8c3176e4bf5050fadb698fc8950de3debcb39845ae876d70594c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"240cbbf928ffa90c9cd7eaa518bcb5617f0337bd136513895f3f88ffa806a31d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"30f9548fde2928aefc70184fe1e870b112c2a770869a78ecd4fa6bd70339673e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"418bb2882cae75fb98d39bf0e4218d01a499f5666e2aafb1810c2da6c9679b94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"9b8f321956f7d59f01cbf02cbc5896e763e304600c0ff079e069d44aa875bee5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"badfd17ab957d573d844010c3b807dde68e8706d0952fdfce991a04abe429721","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ff366311fd43d72666297c0085142c8518628a4849f37fc8ec0089c5423ea5c2","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":20,"runs_affected":6,"failed_runs":6,"run_impact_percent":25,"anchor_occurrences":20,"after_last_push_count":20,"also_in":["DEV","PROD"],"trend_counts":[0,3,10,4,0,2,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-q6mvvg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ec67b8dad40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1ec67cfb92a0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-q6mvvg\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1ec67b8dad40\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x1ec67cfb92a0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9qvdk7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38ab336f65a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x38ab34e123c0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-9qvdk7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x38ab336f65a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x38ab34e123c0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","signature_id":"cb8a33e2b2095072faf50f6e0575b49f9f24cba8d6e5696426aaf71bfa169c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"7254a4758a4c9f3547d8595bb271022e57f6d2108cbb99621ddf2051d8a76f43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114854607929741312","occurred_at":"2026-10-26T23:00:05Z","signature_id":"f6440923d584406c047bf35485df58b43af2e7dc6ec14ef4aa1c454df23ab759","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"5818aadf89cbb8e565ce9ed7e1648b3afa7536f15fa6e2b51f377492bd0f3005","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"7b9ef848d676464d2ee73400ab31d344c132d9313c7ed1cca17c7e9627d47b62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"834c4c46a8192f22e0d48bf91d0668321f2f9d88d2bf7d98f62463f201aee617","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"e335ed94021a55d13e1d24228d347b0dff7fb73437b0d1e2bfe037fa51a527c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"06ae41199949919c239605bcb8f9e2f9e3a8ee881edeb175e6c9ce17aead5070","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"236373cc11d0718c40928c3d4bcb58a691be89266e2b0d292ec91cf3a8719739","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"89783edd71220304494074d29307d2cedf219631d48b35933e55116d70c5cc99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"e31b36627e95d2f3f71c4996bdb62795af4a413809745283f3c40bcaeb6d0f4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113767431116689408","occurred_at":"2026-10-23T23:00:02Z","signature_id":"f8f7c200cdfb6e6274cd28c7b7b9ef53197399b5e9c9e7af2d67108959700998","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"1270c274cd4c10d499af31cc920e02ce39b47ba5d94c893080b01de119b51493","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"640e14fee48b1ca4ff6e655305876438a3843d16dd7030e2bb8a5b187edd7ac1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"6fbd217e3bc9dd89602e36032cdbc664f8536fd0653cbcce8ed06e7055b6823b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"b7e22f3f227aa3cb1e168866cb60c6d1d3f48e89d9b13fd4a3ad06c2734c6a16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"c3ab01bdaf950c018b6e61d884b557f3467426da20857899e19d53decf0357fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"5198038e0006450671f4a020b2979e4a0b24cfb71dd67a1e53a69b2d05885c5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"a950b31532425e9c35f1c63b88d041835751436213ae0ce309be06b9581513d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"fc41f17a1b5ab87a77e0439515a905adf8e88a2e9eeda677d7e7dc84581e9ae1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":10,"after_last_push_count":35,"trend_counts":[0,0,0,0,0,0,10,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-05T23:00:54Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"13447a28327f6cac72eeb38ac4f64daeca2de13dce1a8779d5b023237c94cc0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"48f521861c358da162e56adb1d5a5560a6d110197a1e1ace87d26c0bd91ae752","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"5bf62fc718ddab8012057c407f628773cdec5fa1f4dce294092b41dd220fca46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"5d0bce7e827e2d05c2cc18c9afe55ae2811c60bd7230edf27e23c249ad468b73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"65786b9fd4ff52613cb0353320693acaf1aa58e9e4797f497abddc9ba6d5c20f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"7a2b3613d8cecf72125933f4f03d2d7d31c60d9319bc6e32bde74818fbf4a7c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"80f41f56d6bb15b31124a77cca8ab3bafdbb09260df5d6a0be927f35414248b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"aac52a12f1e11813e645caf10fcf25877853fed7ef788c7c5c5971b57aebcaa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"bdeacb1eac1daccd351ab2768ef041b978c2eb7deaf336006af84dd3f773eae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"d84676500a8d1f8f5aba72d56f9519c4ab3fe5ba3beacd4c258d168261c4440e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ba37c41b19e0f708356e10e680ed08ff4df92d7bbbd8723583a9ec5a0866a3a","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":8.333333333333334,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,2,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-24T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-h45rtr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31f100c0b6e0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x31f100c0b6c0\u003e{\n            observation: \u003c*errors.errorString | 0x31f1027391b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x31f100c0b6a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-h45rtr\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31f100c0b6e0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x31f100c0b6c0\u003e{\n            observation: \u003c*errors.errorString | 0x31f1027391b0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x31f100c0b6a0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-w7wgsb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x80bfcd600c0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x80bfcd60080\u003e{\n            observation: \u003c*errors.errorString | 0x80bfe902e80\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x80bfcd60060\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-21-w7wgsb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x80bfcd600c0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x80bfcd60080\u003e{\n            observation: \u003c*errors.errorString | 0x80bfe902e80\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x80bfcd60060\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-45dcr7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3387fb166020\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x3387fcff27f0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-22-45dcr7\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3387fb166020\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x3387fcff27f0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113812730254200832","occurred_at":"2026-10-24T02:00:02Z","signature_id":"10576e708e91377d8c1278b394efd774f4a486b903e29ebae241b485d49583c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"26cae887797c5ac843b1bbd02abddd07a756cd00aa27c8cd46e6674dee0c40e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2113405158611226624","occurred_at":"2026-10-22T23:00:29Z","signature_id":"f30409415b677de6254b423fb52d7b5d06067065fabc7ba9d473bd0d72b8d191","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0768993bd9d8538b25d85d3aca9a17479e39f0ccf5a24ad0ffd846011729627b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","PROD"],"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T02:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cf97cd86ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-f4qstxjd2dn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\",\n      \"name\": \"1ac5a43d-188f-41a4-873c-672951930233\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:02:55.710362817Z\",\n      \"endTime\": \"2026-10-22T02:32:02.510983362Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\n            correlation ID: 9e15eb20-b5a4-4836-b8fc-ad2c0fd7a951\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cf97cd86ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-f4qstxjd2dn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\",\n      \"name\": \"1ac5a43d-188f-41a4-873c-672951930233\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T02:02:55.710362817Z\",\n      \"endTime\": \"2026-10-22T02:32:02.510983362Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ac5a43d-188f-41a4-873c-672951930233\n            correlation ID: 9e15eb20-b5a4-4836-b8fc-ad2c0fd7a951\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113087989268615168","occurred_at":"2026-10-22T02:00:10Z","signature_id":"49afeb688782238a09a8bf91cfb59ef37a30d3089196924648fb4aaa0757ff46","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"93bb2520ef0f34dd66efce84b14b6252dd8c25360081fb21aba7a8fa22f714a9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","PROD"],"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-22T02:00:10Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-vrx699\" with version 4.22.19 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb37c723e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vrx699: failed waiting for cluster=\"cluster-zstream-4-22-vrx699\" in resourcegroup=\"rg-zstream-upgrade-4-22-mk5975tqzhdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"name\": \"f522dee9-4911-4a39-a8b7-66d53ae810aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T21:21:44.905508951Z\",\n      \"endTime\": \"2026-10-28T21:50:49.351811692Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f522dee9-4911-4a39-a8b7-66d53ae810aa\n            correlation ID: fa068c44-117e-4214-8c6d-206698ccb0ad\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115540873000259584","occurred_at":"2026-10-28T20:27:03Z","signature_id":"851111d98b1137aabdf43323fadae9cc3cc706ab194c4673dfabf5d420922313","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0ece5aa7e8b4a7df4eadead8c6c087ae24930024b2ab923cda14a7ad621f8a51","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-03T00:25:08Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"2a7dcb6ff7dcda026460bada2d10af749d2aaf152cdd767259012c980422e2c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d36338231424797d19366c98a124f0a1acb1e3045326f6f3cc290e8e3b2c20f3","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-24T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1ea319f5c1c8\u003e: \n    resource group \"rg-nightly-5-0-zf68j445hrrz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-5-0-md89qh\" in resourcegroup=\"rg-nightly-5-0-zf68j445hrrz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\",\n      \"name\": \"1895187e-5536-47a1-a44d-a9b6da901fa1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T00:01:49.583919819Z\",\n      \"endTime\": \"2026-10-25T00:25:49.807200592Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\n            correlation ID: 78c885d0-28f3-4f61-bdbb-0c2134f660b2\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1ea319f5c1c8\u003e: \n    resource group \"rg-nightly-5-0-zf68j445hrrz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-5-0-md89qh\" in resourcegroup=\"rg-nightly-5-0-zf68j445hrrz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\",\n      \"name\": \"1895187e-5536-47a1-a44d-a9b6da901fa1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-25T00:01:49.583919819Z\",\n      \"endTime\": \"2026-10-25T00:25:49.807200592Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1895187e-5536-47a1-a44d-a9b6da901fa1\n            correlation ID: 78c885d0-28f3-4f61-bdbb-0c2134f660b2\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2114129822685859840","occurred_at":"2026-10-24T23:00:02Z","signature_id":"236604dc1d677247ddd4d755cb13d2a4dcdf9f9bc20a51fa0133554a3aca5645","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2115217181124857856","occurred_at":"2026-10-27T23:00:49Z","signature_id":"2081f477b523b7866b3f60de53b0fb6ab5d90668ea1f7f412ab3ed026d28073d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ea9bf1d55025e057c393d183143fc354d95b6cff3844a4a67e3e8f63d2f3858d","failure_pattern":"VerifyCustomIngressCertificate timed out after 10m0","search_query":"VerifyCustomIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-25T02:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:525]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x36a1186625c0\u003e: \n    VerifyCustomIngressCertificate timed out after 10m0.003s: ingress route must serve HTTPS with the expected certificate: Get \"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"VerifyCustomIngressCertificate timed out after 10m0.003s: ingress route must serve HTTPS with the expected certificate: Get \\\"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*fmt.wrapError | 0x36a1186623e0\u003e{\n            msg: \"ingress route must serve HTTPS with the expected certificate: Get \\\"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            err: \u003c*url.Error | 0x36a118d74840\u003e{\n                Op: \"Get\",\n                URL: \"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\",\n                Err: \u003c*tls.CertificateVerificationError | 0x36a118d74810\u003e{\n                    UnverifiedCertificates: [\n                        {\n                            Raw: [48, 130, 9, 19, 48, 130, 6, 251, 160, 3, 2, 1, 2, 2, 19, 73, 1, 74, 89, 119, 169, 186, 32, 59, 56, 235, 138, 87, 0, 0, 1, 74, 89, 119, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 30, 23, 13, 50, 54, 49, 48, 50, 53, 48, 49, 53, 54, 49, 48, 90, 23, 13, 50, 55, 48, 50, 48, 50, 48, 48, 53, 54, 49, 48, 90, 48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1, 163, 130, 4, 181, 48, 130, 4, 177, 48, 130, 1, 4, 6, 10, 43, 6, 1, 4, 1, 214, 121, 2, 4, 2, 4, 129, 245, 4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135, 48, 27, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 10, 4, 14, 48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 60, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 7, 4, 47, 48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39, 48, 130, 1, 11, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 129, 254, 48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 110, 6, 3, 85, 29, 17, 4, 103, 48, 101, 130, 64, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 105, 110, 103, 114, 101, 115, 115, 45, 99, 101, 114, 116, 45, 108, 101, 46, 48, 111, 51, 109, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 130, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 129, 241, 6, 3, 85, 29, 31, 4, 129, 233, 48, 129, 230, 48, 129, 227, 160, 129, 224, 160, 129, 221, 134, 108, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 134, 109, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 48, 102, 6, 3, 85, 29, 32, 4, 95, 48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 2, 1, 0, 127, 14, 76, 119, 16, 30, 113, 75, 136, 158, 234, 180, 48, 26, 140, 223, 208, 88, 116, 11, 139, 240, 141, 40, 46, 35, 73, 190, 186, 171, 118, 120, 99, 118, 88, 110, 181, 203, 42, 120, 241, 109, 206, 58, 1, 63, 144, 160, 72, 2, 211, 217, 25, 137, 246, 187, 186, 49, 138, 232, 122, 82, 3, 136, 22, 157, 87, 191, 86, 125, 221, 231, 158, 0, 204, 174, 39, 65, 179, 160, 12, 205, 190, 69, 207, 65, 139, 215, 99, 239, 19, 51, 250, 33, 66, 49, 250, 3, 217, 217, 178, 148, 58, 186, 139, 102, 86, 205, 171, 36, 144, 223, 188, 149, 248, 1, 103, 232, 163, 162, 100, 23, 166, 66, 215, 67, 190, 124, 120, 217, 106, 101, 246, 252, 8, 144, 121, 98, 136, 193, 124, 213, 80, 149, 73, 94, 215, 24, 197, 138, 7, 143, 140, 229, 172, 93, 155, 162, 211, 4, 222, 43, 210, 38, 230, 222, 23, 202, 214, 208, 103, 147, 145, 86, 53, 202, 100, 48, 50, 41, 5, 116, 80, 254, 160, 164, 22, 192, 242, 55, 78, 107, 236, 232, 202, 76, 47, 127, 5, 241, 21, 79, 8, 239, 16, 12, 106, 11, 200, 233, 173, 167, 160, 195, 61, 159, 42, 183, 223, 149, 235, 138, 251, 54, 65, 95, 191, 206, 35, 254, 15, 224, 194, 106, 185, 99, 23, 156, 154, 222, 84, 145, 1, 212, 126, 42, 74, 254, 71, 27, 110, 6, 83, 252, 190, 60, 29, 24, 98, 214, 226, 88, 108, 117, 86, 233, 3, 27, 205, 3, 203, 36, 3, 175, 73, 206, 16, 65, 117, 125, 162, 73, 22, 207, 82, 232, 237, 197, 40, 139, 79, 103, 210, 165, 182, 151, 205, 15, 253, 195, 116, 121, 178, 103, 118, 187, 165, 173, 124, 138, 96, 201, 232, 236, 212, 18, 179, 244, 160, 183, 40, 12, 30, 207, 237, 36, 120, 26, 57, 114, 158, 34, 152, 93, 132, 52, 124, 112, 222, 90, 37, 164, 188, 88, 53, 52, 110, 127, 166, 176, 30, 184, 129, 236, 40, 151, 237, 246, 44, 182, 131, 71, 47, 162, 122, 195, 81, 3, 7, 197, 200, 200, 48, 29, 56, 138, 158, 6, 33, 41, 210, 36, 197, 209, 154, 188, 46, 232, 167, 27, 177, 139, 150, 154, 50, 110, 13, 89, 233, 181, 191, 72, 106, 53, 174, 105, 231, 108, 61, 64, 154, 240, 69, 47, 220, 37, 46, 170, 187, 63, 76, 114, 220, 25, 31, 218, 164, 20, 35, 139, 252, 139, 174, 8, 42, 39, 150, 249, 226, 25, 42, 115, 161, 172, 59, 218, 77, 27, 104, 120, 74, 137, 124, 201, 198, 130, 184, 80, 219, 176, 188, 97, 76, 79, 77, 137, 160, 247, 211, 155, 231, 107, 56, 112, 147, 133, 216, 183, 201, 229, 98, 78, 26, 232, 122, 160, 50, 149, 73, 205, 59, 201, 173, 34, 22, 180, 220, 49, 209, 56, 199, 177, 177, 107, 162, 228, 222, 80, 57, 112, 238, 205],\n                            RawTBSCertificate: [48, 130, 6, 251, 160, 3, 2, 1, 2, 2, 19, 73, 1, 74, 89, 119, 169, 186, 32, 59, 56, 235, 138, 87, 0, 0, 1, 74, 89, 119, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 30, 23, 13, 50, 54, 49, 48, 50, 53, 48, 49, 53, 54, 49, 48, 90, 23, 13, 50, 55, 48, 50, 48, 50, 48, 48, 53, 54, 49, 48, 90, 48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1, 163, 130, 4, 181, 48, 130, 4, 177, 48, 130, 1, 4, 6, 10, 43, 6, 1, 4, 1, 214, 121, 2, 4, 2, 4, 129, 245, 4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135, 48, 27, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 10, 4, 14, 48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 60, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 7, 4, 47, 48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39, 48, 130, 1, 11, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 129, 254, 48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 110, 6, 3, 85, 29, 17, 4, 103, 48, 101, 130, 64, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 105, 110, 103, 114, 101, 115, 115, 45, 99, 101, 114, 116, 45, 108, 101, 46, 48, 111, 51, 109, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 130, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 129, 241, 6, 3, 85, 29, 31, 4, 129, 233, 48, 129, 230, 48, 129, 227, 160, 129, 224, 160, 129, 221, 134, 108, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 134, 109, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 48, 102, 6, 3, 85, 29, 32, 4, 95, 48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109],\n                            RawIssuer: [48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48],\n                            Signature: [127, 14, 76, 119, 16, 30, 113, 75, 136, 158, 234, 180, 48, 26, 140, 223, 208, 88, 116, 11, 139, 240, 141, 40, 46, 35, 73, 190, 186, 171, 118, 120, 99, 118, 88, 110, 181, 203, 42, 120, 241, 109, 206, 58, 1, 63, 144, 160, 72, 2, 211, 217, 25, 137, 246, 187, 186, 49, 138, 232, 122, 82, 3, 136, 22, 157, 87, 191, 86, 125, 221, 231, 158, 0, 204, 174, 39, 65, 179, 160, 12, 205, 190, 69, 207, 65, 139, 215, 99, 239, 19, 51, 250, 33, 66, 49, 250, 3, 217, 217, 178, 148, 58, 186, 139, 102, 86, 205, 171, 36, 144, 223, 188, 149, 248, 1, 103, 232, 163, 162, 100, 23, 166, 66, 215, 67, 190, 124, 120, 217, 106, 101, 246, 252, 8, 144, 121, 98, 136, 193, 124, 213, 80, 149, 73, 94, 215, 24, 197, 138, 7, 143, 140, 229, 172, 93, 155, 162, 211, 4, 222, 43, 210, 38, 230, 222, 23, 202, 214, 208, 103, 147, 145, 86, 53, 202, 100, 48, 50, 41, 5, 116, 80, 254, 160, 164, 22, 192, 242, 55, 78, 107, 236, 232, 202, 76, 47, 127, 5, 241, 21, 79, 8, 239, 16, 12, 106, 11, 200, 233, 173, 167, 160, 195, 61, 159, 42, 183, 223, 149, 235, 138, 251, 54, 65, 95, 191, 206, 35, 254, 15, 224, 194, 106, 185, 99, 23, 156, 154, 222, 84, 145, 1, 212, 126, 42, 74, 254, 71, 27, 110, 6, 83, 252, 190, 60, 29, 24, 98, 214, 226, 88, 108, 117, 86, 233, 3, 27, 205, 3, 203, 36, 3, 175, 73, 206, 16, 65, 117, 125, 162, 73, 22, 207, 82, 232, 237, 197, 40, 139, 79, 103, 210, 165, 182, 151, 205, 15, 253, 195, 116, 121, 178, 103, 118, 187, 165, 173, 124, 138, 96, 201, 232, 236, 212, 18, 179, 244, 160, 183, 40, 12, 30, 207, 237, 36, 120, 26, 57, 114, 158, 34, 152, 93, 132, 52, 124, 112, 222, 90, 37, 164, 188, 88, 53, 52, 110, 127, 166, 176, 30, 184, 129, 236, 40, 151, 237, 246, 44, 182, 131, 71, 47, 162, 122, 195, 81, 3, 7, 197, 200, 200, 48, 29, 56, 138, 158, 6, 33, 41, 210, 36, 197, 209, 154, 188, 46, 232, 167, 27, 177, 139, 150, 154, 50, 110, 13, 89, 233, 181, 191, 72, 106, 53, 174, 105, 231, 108, 61, 64, 154, 240, 69, 47, 220, 37, 46, 170, 187, 63, 76, 114, 220, 25, 31, 218, 164, 20, 35, 139, 252, 139, 174, 8, 42, 39, 150, 249, 226, 25, 42, 115, 161, 172, 59, 218, 77, 27, 104, 120, 74, 137, 124, 201, 198, 130, 184, 80, 219, 176, 188, 97, 76, 79, 77, 137, 160, 247, 211, 155, 231, 107, 56, 112, 147, 133, 216, 183, 201, 229, 98, 78, 26, 232, 122, 160, 50, 149, 73, 205, 59, 201, 173, 34, 22, 180, 220, 49, 209, 56, 199, 177, 177, 107, 162, 228, 222, 80, 57, 112, 238, 205],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a119f65b90\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [3569991644534247369, 3274486715548429274, 3780411684536201245, 13386095367080826074, 4555633294449980078, 8356885564098020936, 8844059572749990990, 2540492176423473444, 5633937353057226705, 779516051327683154, 2327935533544294595, 7836233832775480006, 12443919168731326213, 5381817252368116948, 13149759042380401282, 9307346103102305840, 2490874408770263855, 8905597817602166121, 4800718759162655104, 10753677448201427049, 5736247462418790658, 11125319528290994526, 17747990166117017402, 10636684124628129003, 14904076079720377571, 4842591071448276587, 12095755243242069355, 6630948406570189761, 17118216322786437170, 12282717644381587200, 11394433495304529049, 13509030274166194453],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [9968436300229532023, 6446808008473917675, 4784458],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS G2 RSA CA OCSP 10\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS G2 RSA CA OCSP 10\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: [\"Redmond\"],\n                                Province: [\"WA\"],\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"reserved.aroapp-hcp-stg.azure.com\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"WA\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Redmond\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"reserved.aroapp-hcp-stg.azure.com\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63928490170, loc: nil},\n                            NotAfter: {wall: 0, ext: 63937126570, loc: nil},\n                            KeyUsage: 5,\n                            Extensions: [\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 11129, 2, 4, 2],\n                                    Critical: false,\n                                    Value: [4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 10],\n                                    Critical: false,\n                                    Value: [48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 7],\n                                    Critical: false,\n                                    Value: [48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 5, 160],\n                                },\n                                {\n                                    Id: [2, 5, 29, 17],\n                                    Critical: false,\n                                    Value: \"0e\\x82@*.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\x82!reserved.aroapp-hcp-stg.azure.com\",\n                                },\n                                {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0\\x81\\xe60\\x81㠁ࠁ݆lhttp://www.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\\x86mhttp://crl2.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: false,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122],\n                            AuthorityKeyId: [208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                            OCSPServer: [\n                                \"http://oneocsp.microsoft.com/ocsp\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://www.microsoft.com/pkiops/certs/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010.crt\",\n                                \"http://caissuers.microsoft.com/pkiops/certs/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010.crt\",\n                            ],\n                            DNSNames: [\n                                \"*.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\",\n                                \"reserved.aroapp-hcp-stg.azure.com\",\n                            ],\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://www.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                                \"http://crl2.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                                [1, 3, 6, 1, 4, 1, 311, 76, 509, 1, 1],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                                {\n                                    der: [43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 7, 184, 48, 130, 5, 160, 160, 3, 2, 1, 2, 2, 19, 51, 0, 0, 0, 15, 51, 32, 101, 55, 238, 66, 174, 79, 0, 0, 0, 0, 0, 15, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 56, 48, 49, 50, 48, 48, 51, 48, 51, 90, 23, 13, 50, 57, 48, 54, 48, 51, 50, 48, 48, 51, 48, 51, 90, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1, 163, 130, 2, 129, 48, 130, 2, 125, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 16, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 1, 4, 3, 2, 1, 0, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 25, 6, 9, 43, 6, 1, 4, 1, 130, 55, 20, 2, 4, 12, 30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65, 48, 18, 6, 3, 85, 29, 19, 1, 1, 255, 4, 8, 48, 6, 1, 1, 255, 2, 1, 0, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 129, 171, 6, 3, 85, 29, 31, 4, 129, 163, 48, 129, 160, 48, 129, 157, 160, 129, 154, 160, 129, 151, 134, 73, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 134, 74, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 48, 130, 1, 16, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 130, 1, 2, 48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 2, 1, 0, 53, 25, 202, 233, 168, 117, 225, 216, 202, 253, 37, 193, 59, 82, 225, 231, 158, 43, 148, 226, 244, 58, 210, 79, 218, 6, 34, 82, 26, 1, 24, 227, 162, 186, 183, 183, 28, 36, 50, 117, 13, 103, 68, 125, 157, 74, 130, 75, 250, 227, 68, 233, 70, 246, 0, 69, 9, 194, 82, 233, 103, 37, 151, 199, 183, 139, 90, 119, 124, 64, 30, 148, 227, 188, 140, 109, 56, 41, 188, 39, 125, 18, 18, 61, 203, 203, 228, 196, 70, 226, 165, 1, 192, 16, 209, 32, 250, 14, 3, 228, 67, 219, 38, 135, 37, 172, 200, 160, 179, 4, 106, 94, 240, 67, 63, 15, 123, 245, 206, 35, 195, 183, 201, 190, 251, 222, 155, 125, 147, 179, 135, 62, 223, 198, 219, 61, 201, 244, 172, 113, 0, 210, 164, 101, 129, 54, 89, 159, 28, 54, 229, 137, 209, 122, 153, 74, 123, 85, 75, 194, 213, 140, 208, 121, 1, 121, 149, 15, 57, 67, 139, 179, 10, 123, 146, 213, 90, 16, 24, 118, 233, 29, 13, 111, 50, 49, 244, 189, 209, 22, 241, 168, 163, 103, 242, 14, 154, 112, 12, 227, 150, 175, 3, 244, 109, 189, 20, 121, 106, 54, 195, 241, 182, 115, 187, 198, 11, 213, 38, 70, 136, 84, 69, 112, 170, 204, 31, 61, 139, 111, 227, 228, 23, 166, 129, 0, 251, 125, 131, 88, 128, 226, 242, 60, 59, 163, 24, 98, 19, 54, 103, 255, 79, 226, 229, 111, 106, 103, 230, 232, 90, 100, 58, 160, 69, 157, 63, 23, 12, 102, 135, 240, 129, 39, 52, 32, 176, 53, 36, 100, 40, 118, 67, 122, 116, 152, 226, 43, 128, 9, 10, 188, 176, 185, 7, 71, 41, 40, 46, 125, 224, 119, 65, 164, 140, 54, 120, 186, 51, 29, 86, 142, 217, 16, 53, 137, 89, 131, 0, 105, 206, 183, 120, 189, 237, 175, 130, 63, 206, 200, 137, 50, 152, 89, 8, 93, 225, 1, 150, 88, 196, 101, 25, 125, 38, 133, 157, 16, 150, 229, 149, 197, 17, 206, 130, 111, 23, 6, 113, 21, 15, 44, 191, 55, 50, 82, 21, 104, 180, 149, 142, 83, 93, 151, 237, 227, 219, 19, 62, 193, 49, 177, 121, 147, 115, 125, 114, 59, 27, 76, 79, 6, 73, 21, 132, 115, 243, 29, 90, 81, 44, 164, 212, 88, 224, 25, 191, 53, 72, 41, 96, 153, 171, 234, 123, 66, 3, 190, 156, 61, 25, 247, 71, 91, 11, 172, 130, 87, 170, 13, 189, 255, 34, 96, 48, 223, 44, 79, 87, 211, 37, 213, 218, 253, 142, 6, 81, 78, 5, 53, 120, 152, 56, 163, 39, 49, 111, 49, 154, 169, 170, 77, 172, 58, 68, 104, 96, 112, 104, 244, 83, 107, 205, 178, 45, 174, 38, 24, 135, 223, 140, 87, 181, 248, 82, 193, 86, 231, 192, 177, 77, 95, 10, 85, 107, 20, 43, 84, 251, 61, 169, 198, 189, 96, 7, 248, 176, 244, 75, 227, 238, 171],\n                            RawTBSCertificate: [48, 130, 5, 160, 160, 3, 2, 1, 2, 2, 19, 51, 0, 0, 0, 15, 51, 32, 101, 55, 238, 66, 174, 79, 0, 0, 0, 0, 0, 15, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 56, 48, 49, 50, 48, 48, 51, 48, 51, 90, 23, 13, 50, 57, 48, 54, 48, 51, 50, 48, 48, 51, 48, 51, 90, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1, 163, 130, 2, 129, 48, 130, 2, 125, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 16, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 1, 4, 3, 2, 1, 0, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 25, 6, 9, 43, 6, 1, 4, 1, 130, 55, 20, 2, 4, 12, 30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65, 48, 18, 6, 3, 85, 29, 19, 1, 1, 255, 4, 8, 48, 6, 1, 1, 255, 2, 1, 0, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 129, 171, 6, 3, 85, 29, 31, 4, 129, 163, 48, 129, 160, 48, 129, 157, 160, 129, 154, 160, 129, 151, 134, 73, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 134, 74, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 48, 130, 1, 16, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 130, 1, 2, 48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                            RawSubjectPublicKeyInfo: [48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48],\n                            RawIssuer: [48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [53, 25, 202, 233, 168, 117, 225, 216, 202, 253, 37, 193, 59, 82, 225, 231, 158, 43, 148, 226, 244, 58, 210, 79, 218, 6, 34, 82, 26, 1, 24, 227, 162, 186, 183, 183, 28, 36, 50, 117, 13, 103, 68, 125, 157, 74, 130, 75, 250, 227, 68, 233, 70, 246, 0, 69, 9, 194, 82, 233, 103, 37, 151, 199, 183, 139, 90, 119, 124, 64, 30, 148, 227, 188, 140, 109, 56, 41, 188, 39, 125, 18, 18, 61, 203, 203, 228, 196, 70, 226, 165, 1, 192, 16, 209, 32, 250, 14, 3, 228, 67, 219, 38, 135, 37, 172, 200, 160, 179, 4, 106, 94, 240, 67, 63, 15, 123, 245, 206, 35, 195, 183, 201, 190, 251, 222, 155, 125, 147, 179, 135, 62, 223, 198, 219, 61, 201, 244, 172, 113, 0, 210, 164, 101, 129, 54, 89, 159, 28, 54, 229, 137, 209, 122, 153, 74, 123, 85, 75, 194, 213, 140, 208, 121, 1, 121, 149, 15, 57, 67, 139, 179, 10, 123, 146, 213, 90, 16, 24, 118, 233, 29, 13, 111, 50, 49, 244, 189, 209, 22, 241, 168, 163, 103, 242, 14, 154, 112, 12, 227, 150, 175, 3, 244, 109, 189, 20, 121, 106, 54, 195, 241, 182, 115, 187, 198, 11, 213, 38, 70, 136, 84, 69, 112, 170, 204, 31, 61, 139, 111, 227, 228, 23, 166, 129, 0, 251, 125, 131, 88, 128, 226, 242, 60, 59, 163, 24, 98, 19, 54, 103, 255, 79, 226, 229, 111, 106, 103, 230, 232, 90, 100, 58, 160, 69, 157, 63, 23, 12, 102, 135, 240, 129, 39, 52, 32, 176, 53, 36, 100, 40, 118, 67, 122, 116, 152, 226, 43, 128, 9, 10, 188, 176, 185, 7, 71, 41, 40, 46, 125, 224, 119, 65, 164, 140, 54, 120, 186, 51, 29, 86, 142, 217, 16, 53, 137, 89, 131, 0, 105, 206, 183, 120, 189, 237, 175, 130, 63, 206, 200, 137, 50, 152, 89, 8, 93, 225, 1, 150, 88, 196, 101, 25, 125, 38, 133, 157, 16, 150, 229, 149, 197, 17, 206, 130, 111, 23, 6, 113, 21, 15, 44, 191, 55, 50, 82, 21, 104, 180, 149, 142, 83, 93, 151, 237, 227, 219, 19, 62, 193, 49, 177, 121, 147, 115, 125, 114, 59, 27, 76, 79, 6, 73, 21, 132, 115, 243, 29, 90, 81, 44, 164, 212, 88, 224, 25, 191, 53, 72, 41, 96, 153, 171, 234, 123, 66, 3, 190, 156, 61, 25, 247, 71, 91, 11, 172, 130, 87, 170, 13, 189, 255, 34, 96, 48, 223, 44, 79, 87, 211, 37, 213, 218, 253, 142, 6, 81, 78, 5, 53, 120, 152, 56, 163, 39, 49, 111, 49, 154, 169, 170, 77, 172, 58, 68, 104, 96, 112, 104, 244, 83, 107, 205, 178, 45, 174, 38, 24, 135, 223, 140, 87, 181, 248, 82, 193, 86, 231, 192, 177, 77, 95, 10, 85, 107, 20, 43, 84, 251, 61, 169, 198, 189, 96, 7, 248, 176, 244, 75, 227, 238, 171],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a118ac3670\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [12623528037057212991, 15782976484946675180, 9131881476679932799, 16877507857597638923, 6948446969886104628, 16879364175701064122, 15349600278999781980, 9245340169127444553, 7968545109705329495, 6423711443832766155, 10594494922037898650, 17648935788986680361, 18122500389998752041, 7737602644086511993, 7258855044086458051, 5541026985182791629, 4439073283700877230, 10408130435767131132, 18089869943821258368, 11685763412349126810, 16394152652608075740, 12871239331426121127, 9800942613550528692, 18261315811323797426, 15374412250364208978, 3968124059102095807, 685806708061009512, 7838166407260346135, 16993058175899549691, 15384298262262378419, 3974146097701597185, 13694573362644585378, 13796492994165901152, 13528806942954960401, 3955259362538190321, 13074935763441801879, 2348393719768315797, 11342510827905045171, 130453718204474471, 17672562703128855053, 634844305723830904, 11538538025395239748, 5505257917472087582, 16743812431760803789, 13404336697386206403, 6770939201695035783, 3646446187118575543, 14898275314275200590, 3021489551454039149, 3930692534149328022, 11247669138812385627, 7850138742151524599, 1567520330174748017, 5626181319708709539, 16420623271324550298, 18240848139101269901, 2441811390160163564, 1395980562250961772, 3030591394459285111, 17666945104455592770, 11923213182329091847, 9136006465603430828, 13028421093649262211, 17297287522694218268],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [12560257885759602703, 4278338880794178, 3342336],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS RSA Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS RSA Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS G2 RSA CA OCSP 10\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS G2 RSA CA OCSP 10\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63889675383, loc: nil},\n                            NotAfter: {wall: 0, ext: 64010808183, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 1],\n                                    Critical: false,\n                                    Value: [2, 1, 0],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 20, 2],\n                                    Critical: false,\n                                    Value: [30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65],\n                                },\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 6, 1, 1, 255, 2, 1, 0],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                                },\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0\\x81\\xa00\\x81\\x9d\\xa0\\x81\\x9a\\xa0\\x81\\x97\\x86Ihttp://www.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\\x86Jhttp://crl2.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: 0,\n                            MaxPathLenZero: true,\n                            SubjectKeyId: [208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                            AuthorityKeyId: [222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                            OCSPServer: [\n                                \"http://oneocsp.microsoft.com/ocsp\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://www.microsoft.com/pkiops/certs/Microsoft%20TLS%20RSA%20Root%20G2%20-%20xsign.crt\",\n                                \"http://caissuers.microsoft.com/pkiops/certs/Microsoft%20TLS%20RSA%20Root%20G2%20-%20xsign.crt\",\n                            ],\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://www.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                                \"http://crl2.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 5, 137, 48, 130, 4, 113, 160, 3, 2, 1, 2, 2, 16, 11, 12, 107, 44, 70, 105, 23, 176, 71, 115, 198, 71, 212, 175, 192, 200, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 53, 50, 49, 48, 48, 48, 48, 48, 48, 90, 23, 13, 50, 57, 48, 54, 49, 57, 50, 51, 53, 57, 53, 57, 90, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1, 163, 130, 1, 75, 48, 130, 1, 71, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 118, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 106, 48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116, 48, 66, 6, 3, 85, 29, 31, 4, 59, 48, 57, 48, 55, 160, 53, 160, 51, 134, 49, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 51, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 108, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 1, 1, 0, 11, 188, 180, 43, 55, 116, 197, 75, 167, 54, 2, 54, 198, 175, 225, 19, 204, 138, 154, 151, 27, 252, 227, 68, 140, 238, 84, 0, 229, 228, 65, 26, 70, 240, 11, 118, 52, 255, 153, 85, 98, 53, 170, 173, 138, 156, 12, 90, 98, 204, 67, 192, 203, 234, 74, 29, 189, 107, 29, 175, 177, 25, 37, 102, 156, 247, 121, 110, 91, 37, 128, 36, 129, 133, 199, 121, 45, 53, 95, 66, 9, 120, 237, 159, 226, 101, 213, 196, 32, 80, 130, 16, 143, 106, 58, 237, 24, 142, 72, 137, 171, 166, 9, 248, 225, 128, 252, 6, 117, 22, 39, 37, 90, 143, 184, 58, 112, 108, 24, 71, 240, 99, 244, 83, 14, 217, 21, 53, 32, 243, 145, 61, 19, 1, 160, 226, 250, 36, 8, 69, 86, 119, 209, 103, 169, 49, 66, 208, 87, 255, 26, 38, 190, 197, 67, 216, 87, 169, 99, 255, 175, 243, 58, 216, 246, 123, 75, 16, 162, 151, 161, 207, 102, 190, 103, 103, 9, 249, 181, 76, 231, 171, 25, 43, 191, 34, 61, 125, 222, 198, 247, 239, 99, 56, 44, 221, 86, 196, 249, 9, 82, 218, 223, 227, 136, 111, 213, 109, 107, 247, 8, 65, 47, 142, 62, 134, 49, 159, 241, 124, 74, 245, 214, 97, 164, 70, 70, 40, 239, 87, 186, 176, 111, 146, 216, 210, 119, 37, 237, 123, 166, 41, 206, 59, 114, 121, 17, 212, 224, 167, 84, 109, 53, 43, 164, 122],\n                            RawTBSCertificate: [48, 130, 4, 113, 160, 3, 2, 1, 2, 2, 16, 11, 12, 107, 44, 70, 105, 23, 176, 71, 115, 198, 71, 212, 175, 192, 200, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 53, 50, 49, 48, 48, 48, 48, 48, 48, 90, 23, 13, 50, 57, 48, 54, 49, 57, 50, 51, 53, 57, 53, 57, 90, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1, 163, 130, 1, 75, 48, 130, 1, 71, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 118, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 106, 48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116, 48, 66, 6, 3, 85, 29, 31, 4, 59, 48, 57, 48, 55, 160, 53, 160, 51, 134, 49, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 51, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 108, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                            RawSubjectPublicKeyInfo: [48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50],\n                            RawIssuer: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [11, 188, 180, 43, 55, 116, 197, 75, 167, 54, 2, 54, 198, 175, 225, 19, 204, 138, 154, 151, 27, 252, 227, 68, 140, 238, 84, 0, 229, 228, 65, 26, 70, 240, 11, 118, 52, 255, 153, 85, 98, 53, 170, 173, 138, 156, 12, 90, 98, 204, 67, 192, 203, 234, 74, 29, 189, 107, 29, 175, 177, 25, 37, 102, 156, 247, 121, 110, 91, 37, 128, 36, 129, 133, 199, 121, 45, 53, 95, 66, 9, 120, 237, 159, 226, 101, 213, 196, 32, 80, 130, 16, 143, 106, 58, 237, 24, 142, 72, 137, 171, 166, 9, 248, 225, 128, 252, 6, 117, 22, 39, 37, 90, 143, 184, 58, 112, 108, 24, 71, 240, 99, 244, 83, 14, 217, 21, 53, 32, 243, 145, 61, 19, 1, 160, 226, 250, 36, 8, 69, 86, 119, 209, 103, 169, 49, 66, 208, 87, 255, 26, 38, 190, 197, 67, 216, 87, 169, 99, 255, 175, 243, 58, 216, 246, 123, 75, 16, 162, 151, 161, 207, 102, 190, 103, 103, 9, 249, 181, 76, 231, 171, 25, 43, 191, 34, 61, 125, 222, 198, 247, 239, 99, 56, 44, 221, 86, 196, 249, 9, 82, 218, 223, 227, 136, 111, 213, 109, 107, 247, 8, 65, 47, 142, 62, 134, 49, 159, 241, 124, 74, 245, 214, 97, 164, 70, 70, 40, 239, 87, 186, 176, 111, 146, 216, 210, 119, 37, 237, 123, 166, 41, 206, 59, 114, 121, 17, 212, 224, 167, 84, 109, 53, 43, 164, 122],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a118ac37b0\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [8228224839953229621, 16348041302357558304, 2074854447542410873, 14334067402580427935, 12854688567428618952, 14995722382209926558, 15077658020813214080, 9553354208224609462, 902637906707506152, 15437658075602359723, 4892132923038495724, 14100513645772686328, 15699610158021110198, 116260203274558899, 7107135261377697174, 1346652938663145757, 806906029520556509, 6347563231035665119, 1416154356297248238, 13996791138010853103, 17151436068643035665, 5005973024364225782, 15423091598760460437, 11656337824329587047, 14040304327591079399, 16520357453723561614, 10318202437165161261, 17977527309428632165, 3328466382962324838, 13840171658303184680, 9417131409404627226, 5146490298140267738, 6849551933690625799, 10139571647697603240, 3552143779013315793, 15996764607220205453, 9355211167345930524, 7804203173400544275, 12256789870797553077, 5196564742993393824, 12236288164267916870, 9528338169550421684, 3418034452713204002, 17138102837348087434, 15605821747820856657, 6942198289040656341, 6027260588802198758, 4685154310243513825, 1772230202496746547, 12898472891819094090, 6838761955263670191, 15563102318165378361, 5970264090691015160, 5437520332982603245, 15757213624540617421, 6308098459188714935, 3452756883569783574, 995879279813522815, 6719559476972000178, 3606112178562803563, 6954603544383871929, 16027092704045215034, 8191838635022848599, 16134862131230222352],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [5148676810827874504, 796129072041760688],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS RSA Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS RSA Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63883382400, loc: nil},\n                            NotAfter: {wall: 0, ext: 64012204799, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 3, 1, 1, 255],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116],\n                                },\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0907\\xa05\\xa03\\x861http://crl3.digicert.com/DigiCertGlobalRootG2.crl\",\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                            AuthorityKeyId: [78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            OCSPServer: [\n                                \"http://ocsp.digicert.com\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://cacerts.digicert.com/DigiCertGlobalRootG2.crt\",\n                            ],\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://crl3.digicert.com/DigiCertGlobalRootG2.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 3, 142, 48, 130, 2, 118, 160, 3, 2, 1, 2, 2, 16, 3, 58, 241, 230, 167, 17, 169, 160, 187, 40, 100, 177, 29, 9, 250, 229, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 49, 51, 48, 56, 48, 49, 49, 50, 48, 48, 48, 48, 90, 23, 13, 51, 56, 48, 49, 49, 53, 49, 50, 48, 48, 48, 48, 90, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1, 163, 66, 48, 64, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 96, 103, 40, 148, 111, 14, 72, 99, 235, 49, 221, 234, 103, 24, 213, 137, 125, 60, 197, 139, 74, 127, 233, 190, 219, 43, 23, 223, 176, 95, 115, 119, 42, 50, 19, 57, 129, 103, 66, 132, 35, 242, 69, 103, 53, 236, 136, 191, 248, 143, 176, 97, 12, 52, 164, 174, 32, 76, 132, 198, 219, 248, 53, 225, 118, 217, 223, 166, 66, 187, 199, 68, 8, 134, 127, 54, 116, 36, 90, 218, 108, 13, 20, 89, 53, 189, 242, 73, 221, 182, 31, 201, 179, 13, 71, 42, 61, 153, 47, 187, 92, 187, 181, 212, 32, 225, 153, 95, 83, 70, 21, 219, 104, 155, 240, 243, 48, 213, 62, 49, 226, 141, 132, 158, 227, 138, 218, 218, 150, 62, 53, 19, 165, 95, 240, 249, 112, 80, 112, 71, 65, 17, 87, 25, 78, 192, 143, 174, 6, 196, 149, 19, 23, 47, 27, 37, 159, 117, 242, 177, 142, 153, 161, 111, 19, 177, 65, 113, 254, 136, 42, 200, 79, 16, 32, 85, 215, 243, 20, 69, 229, 224, 68, 244, 234, 135, 149, 50, 147, 14, 254, 83, 70, 250, 44, 157, 255, 139, 34, 185, 75, 217, 9, 69, 164, 222, 164, 184, 154, 88, 221, 27, 125, 82, 159, 142, 89, 67, 136, 129, 164, 158, 38, 213, 111, 173, 221, 13, 198, 55, 125, 237, 3, 146, 27, 229, 119, 95, 118, 238, 60, 141, 196, 93, 86, 91, 162, 217, 102, 110, 179, 53, 55, 229, 50, 182],\n                            RawTBSCertificate: [48, 130, 2, 118, 160, 3, 2, 1, 2, 2, 16, 3, 58, 241, 230, 167, 17, 169, 160, 187, 40, 100, 177, 29, 9, 250, 229, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 49, 51, 48, 56, 48, 49, 49, 50, 48, 48, 48, 48, 90, 23, 13, 51, 56, 48, 49, 49, 53, 49, 50, 48, 48, 48, 48, 90, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1, 163, 66, 48, 64, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            RawIssuer: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [96, 103, 40, 148, 111, 14, 72, 99, 235, 49, 221, 234, 103, 24, 213, 137, 125, 60, 197, 139, 74, 127, 233, 190, 219, 43, 23, 223, 176, 95, 115, 119, 42, 50, 19, 57, 129, 103, 66, 132, 35, 242, 69, 103, 53, 236, 136, 191, 248, 143, 176, 97, 12, 52, 164, 174, 32, 76, 132, 198, 219, 248, 53, 225, 118, 217, 223, 166, 66, 187, 199, 68, 8, 134, 127, 54, 116, 36, 90, 218, 108, 13, 20, 89, 53, 189, 242, 73, 221, 182, 31, 201, 179, 13, 71, 42, 61, 153, 47, 187, 92, 187, 181, 212, 32, 225, 153, 95, 83, 70, 21, 219, 104, 155, 240, 243, 48, 213, 62, 49, 226, 141, 132, 158, 227, 138, 218, 218, 150, 62, 53, 19, 165, 95, 240, 249, 112, 80, 112, 71, 65, 17, 87, 25, 78, 192, 143, 174, 6, 196, 149, 19, 23, 47, 27, 37, 159, 117, 242, 177, 142, 153, 161, 111, 19, 177, 65, 113, 254, 136, 42, 200, 79, 16, 32, 85, 215, 243, 20, 69, 229, 224, 68, 244, 234, 135, 149, 50, 147, 14, 254, 83, 70, 250, 44, 157, 255, 139, 34, 185, 75, 217, 9, 69, 164, 222, 164, 184, 154, 88, 221, 27, 125, 82, 159, 142, 89, 67, 136, 129, 164, 158, 38, 213, 111, 173, 221, 13, 198, 55, 125, 237, 3, 146, 27, 229, 119, 95, 118, 238, 60, 141, 196, 93, 86, 91, 162, 217, 102, 110, 179, 53, 55, 229, 50, 182],\n                            SignatureAlgorithm: 4,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a11782c840\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [13518897546323503493, 12614377001919816122, 8557832239203824717, 7433462159093546310, 9504020990483827396, 702024576034523901, 5333964605457796894, 8971906913408307990, 18252057450268743444, 5039799191733612724, 10346373305228545133, 18107087016177239977, 3285062882728703971, 2137966295034491701, 13110919595711778188, 17490549066068800107, 17827376715711028367, 6958731023901346072, 11071174057932843776, 11902983757762476756, 4850057369827327278, 16198203776063958047, 2241017827087547863, 13230558770443115738, 9142400105587256400, 4952379155939763112, 7768617195409377572, 9912770753372816922, 14554377099045433914, 13412012197143714132, 12855684210282069830, 13490476835732679625],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [13486139796020132581, 232764303710726560],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63510955200, loc: nil},\n                            NotAfter: {wall: 0, ext: 64282766400, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 3, 1, 1, 255],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: nil,\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            AuthorityKeyId: nil,\n                            OCSPServer: nil,\n                            IssuingCertificateURL: nil,\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: nil,\n                            PolicyIdentifiers: nil,\n                            Policies: nil,\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                    ],\n                    Err: \u003cx509.UnknownAuthorityError\u003e{\n                        Cert: 0x36a1182f5208 (cyclic reference),\n                        hintErr: nil,\n                        hintCert: nil,\n                    },\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:525]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x36a1186625c0\u003e: \n    VerifyCustomIngressCertificate timed out after 10m0.003s: ingress route must serve HTTPS with the expected certificate: Get \"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        msg: \"VerifyCustomIngressCertificate timed out after 10m0.003s: ingress route must serve HTTPS with the expected certificate: Get \\\"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n        err: \u003c*fmt.wrapError | 0x36a1186623e0\u003e{\n            msg: \"ingress route must serve HTTPS with the expected certificate: Get \\\"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            err: \u003c*url.Error | 0x36a118d74840\u003e{\n                Op: \"Get\",\n                URL: \"https://agnhost-e2e-sample-app-l49xr.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\",\n                Err: \u003c*tls.CertificateVerificationError | 0x36a118d74810\u003e{\n                    UnverifiedCertificates: [\n                        {\n                            Raw: [48, 130, 9, 19, 48, 130, 6, 251, 160, 3, 2, 1, 2, 2, 19, 73, 1, 74, 89, 119, 169, 186, 32, 59, 56, 235, 138, 87, 0, 0, 1, 74, 89, 119, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 30, 23, 13, 50, 54, 49, 48, 50, 53, 48, 49, 53, 54, 49, 48, 90, 23, 13, 50, 55, 48, 50, 48, 50, 48, 48, 53, 54, 49, 48, 90, 48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1, 163, 130, 4, 181, 48, 130, 4, 177, 48, 130, 1, 4, 6, 10, 43, 6, 1, 4, 1, 214, 121, 2, 4, 2, 4, 129, 245, 4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135, 48, 27, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 10, 4, 14, 48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 60, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 7, 4, 47, 48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39, 48, 130, 1, 11, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 129, 254, 48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 110, 6, 3, 85, 29, 17, 4, 103, 48, 101, 130, 64, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 105, 110, 103, 114, 101, 115, 115, 45, 99, 101, 114, 116, 45, 108, 101, 46, 48, 111, 51, 109, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 130, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 129, 241, 6, 3, 85, 29, 31, 4, 129, 233, 48, 129, 230, 48, 129, 227, 160, 129, 224, 160, 129, 221, 134, 108, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 134, 109, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 48, 102, 6, 3, 85, 29, 32, 4, 95, 48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 2, 1, 0, 127, 14, 76, 119, 16, 30, 113, 75, 136, 158, 234, 180, 48, 26, 140, 223, 208, 88, 116, 11, 139, 240, 141, 40, 46, 35, 73, 190, 186, 171, 118, 120, 99, 118, 88, 110, 181, 203, 42, 120, 241, 109, 206, 58, 1, 63, 144, 160, 72, 2, 211, 217, 25, 137, 246, 187, 186, 49, 138, 232, 122, 82, 3, 136, 22, 157, 87, 191, 86, 125, 221, 231, 158, 0, 204, 174, 39, 65, 179, 160, 12, 205, 190, 69, 207, 65, 139, 215, 99, 239, 19, 51, 250, 33, 66, 49, 250, 3, 217, 217, 178, 148, 58, 186, 139, 102, 86, 205, 171, 36, 144, 223, 188, 149, 248, 1, 103, 232, 163, 162, 100, 23, 166, 66, 215, 67, 190, 124, 120, 217, 106, 101, 246, 252, 8, 144, 121, 98, 136, 193, 124, 213, 80, 149, 73, 94, 215, 24, 197, 138, 7, 143, 140, 229, 172, 93, 155, 162, 211, 4, 222, 43, 210, 38, 230, 222, 23, 202, 214, 208, 103, 147, 145, 86, 53, 202, 100, 48, 50, 41, 5, 116, 80, 254, 160, 164, 22, 192, 242, 55, 78, 107, 236, 232, 202, 76, 47, 127, 5, 241, 21, 79, 8, 239, 16, 12, 106, 11, 200, 233, 173, 167, 160, 195, 61, 159, 42, 183, 223, 149, 235, 138, 251, 54, 65, 95, 191, 206, 35, 254, 15, 224, 194, 106, 185, 99, 23, 156, 154, 222, 84, 145, 1, 212, 126, 42, 74, 254, 71, 27, 110, 6, 83, 252, 190, 60, 29, 24, 98, 214, 226, 88, 108, 117, 86, 233, 3, 27, 205, 3, 203, 36, 3, 175, 73, 206, 16, 65, 117, 125, 162, 73, 22, 207, 82, 232, 237, 197, 40, 139, 79, 103, 210, 165, 182, 151, 205, 15, 253, 195, 116, 121, 178, 103, 118, 187, 165, 173, 124, 138, 96, 201, 232, 236, 212, 18, 179, 244, 160, 183, 40, 12, 30, 207, 237, 36, 120, 26, 57, 114, 158, 34, 152, 93, 132, 52, 124, 112, 222, 90, 37, 164, 188, 88, 53, 52, 110, 127, 166, 176, 30, 184, 129, 236, 40, 151, 237, 246, 44, 182, 131, 71, 47, 162, 122, 195, 81, 3, 7, 197, 200, 200, 48, 29, 56, 138, 158, 6, 33, 41, 210, 36, 197, 209, 154, 188, 46, 232, 167, 27, 177, 139, 150, 154, 50, 110, 13, 89, 233, 181, 191, 72, 106, 53, 174, 105, 231, 108, 61, 64, 154, 240, 69, 47, 220, 37, 46, 170, 187, 63, 76, 114, 220, 25, 31, 218, 164, 20, 35, 139, 252, 139, 174, 8, 42, 39, 150, 249, 226, 25, 42, 115, 161, 172, 59, 218, 77, 27, 104, 120, 74, 137, 124, 201, 198, 130, 184, 80, 219, 176, 188, 97, 76, 79, 77, 137, 160, 247, 211, 155, 231, 107, 56, 112, 147, 133, 216, 183, 201, 229, 98, 78, 26, 232, 122, 160, 50, 149, 73, 205, 59, 201, 173, 34, 22, 180, 220, 49, 209, 56, 199, 177, 177, 107, 162, 228, 222, 80, 57, 112, 238, 205],\n                            RawTBSCertificate: [48, 130, 6, 251, 160, 3, 2, 1, 2, 2, 19, 73, 1, 74, 89, 119, 169, 186, 32, 59, 56, 235, 138, 87, 0, 0, 1, 74, 89, 119, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 30, 23, 13, 50, 54, 49, 48, 50, 53, 48, 49, 53, 54, 49, 48, 90, 23, 13, 50, 55, 48, 50, 48, 50, 48, 48, 53, 54, 49, 48, 90, 48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1, 163, 130, 4, 181, 48, 130, 4, 177, 48, 130, 1, 4, 6, 10, 43, 6, 1, 4, 1, 214, 121, 2, 4, 2, 4, 129, 245, 4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135, 48, 27, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 10, 4, 14, 48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 60, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 7, 4, 47, 48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39, 48, 130, 1, 11, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 129, 254, 48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 110, 6, 3, 85, 29, 17, 4, 103, 48, 101, 130, 64, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 105, 110, 103, 114, 101, 115, 115, 45, 99, 101, 114, 116, 45, 108, 101, 46, 48, 111, 51, 109, 46, 117, 107, 115, 111, 117, 116, 104, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 130, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 129, 241, 6, 3, 85, 29, 31, 4, 129, 233, 48, 129, 230, 48, 129, 227, 160, 129, 224, 160, 129, 221, 134, 108, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 134, 109, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 112, 97, 114, 116, 105, 116, 105, 111, 110, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 95, 80, 97, 114, 116, 105, 116, 105, 111, 110, 48, 48, 48, 57, 57, 46, 99, 114, 108, 48, 102, 6, 3, 85, 29, 32, 4, 95, 48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                            RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 121, 183, 117, 223, 172, 129, 21, 158, 33, 40, 228, 200, 112, 100, 153, 170, 116, 250, 158, 245, 201, 135, 0, 237, 144, 31, 12, 163, 250, 168, 50, 92, 5, 220, 113, 137, 143, 3, 193, 167, 220, 193, 78, 195, 131, 29, 107, 67, 52, 86, 236, 30, 39, 30, 107, 206, 213, 234, 58, 181, 33, 44, 227, 147, 157, 23, 219, 20, 127, 20, 235, 246, 77, 135, 5, 240, 135, 231, 58, 154, 101, 19, 42, 160, 128, 73, 94, 79, 155, 62, 172, 213, 53, 57, 2, 149, 60, 188, 169, 233, 44, 216, 105, 66, 159, 148, 70, 178, 113, 177, 128, 123, 151, 10, 7, 254, 171, 205, 105, 34, 145, 93, 19, 218, 220, 219, 47, 129, 42, 85, 247, 229, 7, 118, 48, 182, 125, 84, 45, 187, 16, 138, 130, 74, 176, 14, 70, 229, 120, 108, 212, 172, 177, 174, 141, 161, 231, 167, 5, 108, 191, 229, 39, 27, 122, 242, 198, 32, 78, 125, 8, 168, 234, 200, 195, 10, 209, 101, 183, 248, 26, 10, 82, 78, 47, 196, 44, 55, 199, 207, 209, 35, 65, 164, 44, 162, 97, 81, 36, 122, 188, 105, 82, 214, 39, 252, 78, 115, 249, 159, 36, 212, 167, 6, 72, 63, 56, 220, 88, 50, 162, 50, 174, 185, 196, 246, 208, 84, 90, 212, 218, 52, 118, 184, 101, 201, 132, 48, 29, 45, 113, 80, 81, 157, 255, 235, 218, 49, 139, 40, 124, 111, 147, 143, 201, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 120, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 11, 48, 9, 6, 3, 85, 4, 8, 19, 2, 87, 65, 49, 16, 48, 14, 6, 3, 85, 4, 7, 19, 7, 82, 101, 100, 109, 111, 110, 100, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 42, 48, 40, 6, 3, 85, 4, 3, 19, 33, 114, 101, 115, 101, 114, 118, 101, 100, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 45, 115, 116, 103, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109],\n                            RawIssuer: [48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48],\n                            Signature: [127, 14, 76, 119, 16, 30, 113, 75, 136, 158, 234, 180, 48, 26, 140, 223, 208, 88, 116, 11, 139, 240, 141, 40, 46, 35, 73, 190, 186, 171, 118, 120, 99, 118, 88, 110, 181, 203, 42, 120, 241, 109, 206, 58, 1, 63, 144, 160, 72, 2, 211, 217, 25, 137, 246, 187, 186, 49, 138, 232, 122, 82, 3, 136, 22, 157, 87, 191, 86, 125, 221, 231, 158, 0, 204, 174, 39, 65, 179, 160, 12, 205, 190, 69, 207, 65, 139, 215, 99, 239, 19, 51, 250, 33, 66, 49, 250, 3, 217, 217, 178, 148, 58, 186, 139, 102, 86, 205, 171, 36, 144, 223, 188, 149, 248, 1, 103, 232, 163, 162, 100, 23, 166, 66, 215, 67, 190, 124, 120, 217, 106, 101, 246, 252, 8, 144, 121, 98, 136, 193, 124, 213, 80, 149, 73, 94, 215, 24, 197, 138, 7, 143, 140, 229, 172, 93, 155, 162, 211, 4, 222, 43, 210, 38, 230, 222, 23, 202, 214, 208, 103, 147, 145, 86, 53, 202, 100, 48, 50, 41, 5, 116, 80, 254, 160, 164, 22, 192, 242, 55, 78, 107, 236, 232, 202, 76, 47, 127, 5, 241, 21, 79, 8, 239, 16, 12, 106, 11, 200, 233, 173, 167, 160, 195, 61, 159, 42, 183, 223, 149, 235, 138, 251, 54, 65, 95, 191, 206, 35, 254, 15, 224, 194, 106, 185, 99, 23, 156, 154, 222, 84, 145, 1, 212, 126, 42, 74, 254, 71, 27, 110, 6, 83, 252, 190, 60, 29, 24, 98, 214, 226, 88, 108, 117, 86, 233, 3, 27, 205, 3, 203, 36, 3, 175, 73, 206, 16, 65, 117, 125, 162, 73, 22, 207, 82, 232, 237, 197, 40, 139, 79, 103, 210, 165, 182, 151, 205, 15, 253, 195, 116, 121, 178, 103, 118, 187, 165, 173, 124, 138, 96, 201, 232, 236, 212, 18, 179, 244, 160, 183, 40, 12, 30, 207, 237, 36, 120, 26, 57, 114, 158, 34, 152, 93, 132, 52, 124, 112, 222, 90, 37, 164, 188, 88, 53, 52, 110, 127, 166, 176, 30, 184, 129, 236, 40, 151, 237, 246, 44, 182, 131, 71, 47, 162, 122, 195, 81, 3, 7, 197, 200, 200, 48, 29, 56, 138, 158, 6, 33, 41, 210, 36, 197, 209, 154, 188, 46, 232, 167, 27, 177, 139, 150, 154, 50, 110, 13, 89, 233, 181, 191, 72, 106, 53, 174, 105, 231, 108, 61, 64, 154, 240, 69, 47, 220, 37, 46, 170, 187, 63, 76, 114, 220, 25, 31, 218, 164, 20, 35, 139, 252, 139, 174, 8, 42, 39, 150, 249, 226, 25, 42, 115, 161, 172, 59, 218, 77, 27, 104, 120, 74, 137, 124, 201, 198, 130, 184, 80, 219, 176, 188, 97, 76, 79, 77, 137, 160, 247, 211, 155, 231, 107, 56, 112, 147, 133, 216, 183, 201, 229, 98, 78, 26, 232, 122, 160, 50, 149, 73, 205, 59, 201, 173, 34, 22, 180, 220, 49, 209, 56, 199, 177, 177, 107, 162, 228, 222, 80, 57, 112, 238, 205],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a119f65b90\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [3569991644534247369, 3274486715548429274, 3780411684536201245, 13386095367080826074, 4555633294449980078, 8356885564098020936, 8844059572749990990, 2540492176423473444, 5633937353057226705, 779516051327683154, 2327935533544294595, 7836233832775480006, 12443919168731326213, 5381817252368116948, 13149759042380401282, 9307346103102305840, 2490874408770263855, 8905597817602166121, 4800718759162655104, 10753677448201427049, 5736247462418790658, 11125319528290994526, 17747990166117017402, 10636684124628129003, 14904076079720377571, 4842591071448276587, 12095755243242069355, 6630948406570189761, 17118216322786437170, 12282717644381587200, 11394433495304529049, 13509030274166194453],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [9968436300229532023, 6446808008473917675, 4784458],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS G2 RSA CA OCSP 10\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS G2 RSA CA OCSP 10\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: [\"Redmond\"],\n                                Province: [\"WA\"],\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"reserved.aroapp-hcp-stg.azure.com\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"WA\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Redmond\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"reserved.aroapp-hcp-stg.azure.com\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63928490170, loc: nil},\n                            NotAfter: {wall: 0, ext: 63937126570, loc: nil},\n                            KeyUsage: 5,\n                            Extensions: [\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 11129, 2, 4, 2],\n                                    Critical: false,\n                                    Value: [4, 129, 242, 0, 240, 0, 118, 0, 68, 194, 189, 12, 233, 20, 14, 100, 165, 201, 74, 1, 147, 10, 90, 161, 187, 53, 151, 14, 0, 238, 17, 22, 137, 104, 42, 28, 68, 215, 181, 102, 0, 0, 1, 161, 112, 206, 239, 163, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 172, 23, 89, 180, 174, 255, 10, 75, 120, 126, 2, 114, 141, 52, 53, 9, 237, 84, 209, 98, 89, 54, 179, 9, 54, 118, 245, 178, 107, 207, 5, 69, 2, 32, 89, 15, 78, 239, 171, 216, 123, 169, 212, 117, 5, 12, 137, 15, 63, 83, 41, 96, 223, 14, 136, 244, 40, 31, 142, 25, 69, 157, 200, 47, 205, 167, 0, 118, 0, 0, 26, 93, 26, 28, 45, 147, 117, 182, 72, 85, 120, 248, 47, 113, 161, 174, 110, 239, 57, 125, 41, 124, 138, 227, 21, 123, 202, 222, 225, 160, 30, 0, 0, 1, 161, 112, 206, 240, 23, 0, 0, 4, 3, 0, 71, 48, 69, 2, 33, 0, 209, 5, 94, 127, 36, 105, 13, 219, 166, 20, 49, 161, 208, 24, 148, 233, 208, 121, 79, 240, 195, 85, 152, 81, 108, 234, 202, 116, 68, 189, 194, 57, 2, 32, 65, 141, 139, 86, 66, 203, 70, 108, 26, 159, 237, 96, 32, 237, 241, 53, 230, 112, 171, 119, 90, 74, 10, 14, 99, 1, 225, 78, 245, 158, 165, 135],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 10],\n                                    Critical: false,\n                                    Value: [48, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 7],\n                                    Critical: false,\n                                    Value: [48, 45, 6, 37, 43, 6, 1, 4, 1, 130, 55, 21, 8, 135, 189, 215, 27, 129, 231, 235, 70, 130, 129, 157, 46, 142, 208, 12, 135, 240, 218, 29, 93, 133, 246, 239, 42, 129, 232, 226, 53, 2, 1, 100, 2, 1, 39],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 129, 251, 48, 97, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 85, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 103, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 91, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 71, 50, 37, 50, 48, 82, 83, 65, 37, 50, 48, 67, 65, 37, 50, 48, 79, 67, 83, 80, 37, 50, 48, 49, 48, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 5, 160],\n                                },\n                                {\n                                    Id: [2, 5, 29, 17],\n                                    Critical: false,\n                                    Value: \"0e\\x82@*.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\\x82!reserved.aroapp-hcp-stg.azure.com\",\n                                },\n                                {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0\\x81\\xe60\\x81㠁ࠁ݆lhttp://www.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\\x86mhttp://crl2.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 93, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 81, 6, 12, 43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1, 48, 65, 48, 63, 6, 8, 43, 6, 1, 5, 5, 7, 2, 1, 22, 51, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 68, 111, 99, 115, 47, 82, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 104, 116, 109],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: false,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [138, 66, 241, 22, 211, 0, 191, 156, 21, 196, 84, 38, 230, 58, 211, 190, 115, 187, 163, 122],\n                            AuthorityKeyId: [208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                            OCSPServer: [\n                                \"http://oneocsp.microsoft.com/ocsp\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://www.microsoft.com/pkiops/certs/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010.crt\",\n                                \"http://caissuers.microsoft.com/pkiops/certs/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010.crt\",\n                            ],\n                            DNSNames: [\n                                \"*.apps.aro.ingress-cert-le.0o3m.uksouth.aroapp-hcp-stg.azure.com\",\n                                \"reserved.aroapp-hcp-stg.azure.com\",\n                            ],\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://www.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                                \"http://crl2.microsoft.com/pkiops/crl/partition/Microsoft%20TLS%20G2%20RSA%20CA%20OCSP%2010_Partition00099.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                                [1, 3, 6, 1, 4, 1, 311, 76, 509, 1, 1],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                                {\n                                    der: [43, 6, 1, 4, 1, 130, 55, 76, 131, 125, 1, 1],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 7, 184, 48, 130, 5, 160, 160, 3, 2, 1, 2, 2, 19, 51, 0, 0, 0, 15, 51, 32, 101, 55, 238, 66, 174, 79, 0, 0, 0, 0, 0, 15, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 56, 48, 49, 50, 48, 48, 51, 48, 51, 90, 23, 13, 50, 57, 48, 54, 48, 51, 50, 48, 48, 51, 48, 51, 90, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1, 163, 130, 2, 129, 48, 130, 2, 125, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 16, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 1, 4, 3, 2, 1, 0, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 25, 6, 9, 43, 6, 1, 4, 1, 130, 55, 20, 2, 4, 12, 30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65, 48, 18, 6, 3, 85, 29, 19, 1, 1, 255, 4, 8, 48, 6, 1, 1, 255, 2, 1, 0, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 129, 171, 6, 3, 85, 29, 31, 4, 129, 163, 48, 129, 160, 48, 129, 157, 160, 129, 154, 160, 129, 151, 134, 73, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 134, 74, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 48, 130, 1, 16, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 130, 1, 2, 48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 2, 1, 0, 53, 25, 202, 233, 168, 117, 225, 216, 202, 253, 37, 193, 59, 82, 225, 231, 158, 43, 148, 226, 244, 58, 210, 79, 218, 6, 34, 82, 26, 1, 24, 227, 162, 186, 183, 183, 28, 36, 50, 117, 13, 103, 68, 125, 157, 74, 130, 75, 250, 227, 68, 233, 70, 246, 0, 69, 9, 194, 82, 233, 103, 37, 151, 199, 183, 139, 90, 119, 124, 64, 30, 148, 227, 188, 140, 109, 56, 41, 188, 39, 125, 18, 18, 61, 203, 203, 228, 196, 70, 226, 165, 1, 192, 16, 209, 32, 250, 14, 3, 228, 67, 219, 38, 135, 37, 172, 200, 160, 179, 4, 106, 94, 240, 67, 63, 15, 123, 245, 206, 35, 195, 183, 201, 190, 251, 222, 155, 125, 147, 179, 135, 62, 223, 198, 219, 61, 201, 244, 172, 113, 0, 210, 164, 101, 129, 54, 89, 159, 28, 54, 229, 137, 209, 122, 153, 74, 123, 85, 75, 194, 213, 140, 208, 121, 1, 121, 149, 15, 57, 67, 139, 179, 10, 123, 146, 213, 90, 16, 24, 118, 233, 29, 13, 111, 50, 49, 244, 189, 209, 22, 241, 168, 163, 103, 242, 14, 154, 112, 12, 227, 150, 175, 3, 244, 109, 189, 20, 121, 106, 54, 195, 241, 182, 115, 187, 198, 11, 213, 38, 70, 136, 84, 69, 112, 170, 204, 31, 61, 139, 111, 227, 228, 23, 166, 129, 0, 251, 125, 131, 88, 128, 226, 242, 60, 59, 163, 24, 98, 19, 54, 103, 255, 79, 226, 229, 111, 106, 103, 230, 232, 90, 100, 58, 160, 69, 157, 63, 23, 12, 102, 135, 240, 129, 39, 52, 32, 176, 53, 36, 100, 40, 118, 67, 122, 116, 152, 226, 43, 128, 9, 10, 188, 176, 185, 7, 71, 41, 40, 46, 125, 224, 119, 65, 164, 140, 54, 120, 186, 51, 29, 86, 142, 217, 16, 53, 137, 89, 131, 0, 105, 206, 183, 120, 189, 237, 175, 130, 63, 206, 200, 137, 50, 152, 89, 8, 93, 225, 1, 150, 88, 196, 101, 25, 125, 38, 133, 157, 16, 150, 229, 149, 197, 17, 206, 130, 111, 23, 6, 113, 21, 15, 44, 191, 55, 50, 82, 21, 104, 180, 149, 142, 83, 93, 151, 237, 227, 219, 19, 62, 193, 49, 177, 121, 147, 115, 125, 114, 59, 27, 76, 79, 6, 73, 21, 132, 115, 243, 29, 90, 81, 44, 164, 212, 88, 224, 25, 191, 53, 72, 41, 96, 153, 171, 234, 123, 66, 3, 190, 156, 61, 25, 247, 71, 91, 11, 172, 130, 87, 170, 13, 189, 255, 34, 96, 48, 223, 44, 79, 87, 211, 37, 213, 218, 253, 142, 6, 81, 78, 5, 53, 120, 152, 56, 163, 39, 49, 111, 49, 154, 169, 170, 77, 172, 58, 68, 104, 96, 112, 104, 244, 83, 107, 205, 178, 45, 174, 38, 24, 135, 223, 140, 87, 181, 248, 82, 193, 86, 231, 192, 177, 77, 95, 10, 85, 107, 20, 43, 84, 251, 61, 169, 198, 189, 96, 7, 248, 176, 244, 75, 227, 238, 171],\n                            RawTBSCertificate: [48, 130, 5, 160, 160, 3, 2, 1, 2, 2, 19, 51, 0, 0, 0, 15, 51, 32, 101, 55, 238, 66, 174, 79, 0, 0, 0, 0, 0, 15, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 56, 48, 49, 50, 48, 48, 51, 48, 51, 90, 23, 13, 50, 57, 48, 54, 48, 51, 50, 48, 48, 51, 48, 51, 90, 48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1, 163, 130, 2, 129, 48, 130, 2, 125, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 16, 6, 9, 43, 6, 1, 4, 1, 130, 55, 21, 1, 4, 3, 2, 1, 0, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 25, 6, 9, 43, 6, 1, 4, 1, 130, 55, 20, 2, 4, 12, 30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65, 48, 18, 6, 3, 85, 29, 19, 1, 1, 255, 4, 8, 48, 6, 1, 1, 255, 2, 1, 0, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 129, 171, 6, 3, 85, 29, 31, 4, 129, 163, 48, 129, 160, 48, 129, 157, 160, 129, 154, 160, 129, 151, 134, 73, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 134, 74, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 50, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 114, 108, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 46, 99, 114, 108, 48, 130, 1, 16, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 130, 1, 2, 48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                            RawSubjectPublicKeyInfo: [48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 240, 12, 79, 91, 95, 252, 66, 28, 180, 206, 63, 245, 7, 43, 166, 131, 126, 201, 157, 121, 207, 246, 233, 172, 165, 119, 195, 55, 169, 250, 247, 7, 245, 45, 152, 248, 17, 102, 235, 66, 42, 14, 210, 209, 6, 25, 246, 119, 19, 95, 132, 236, 214, 81, 147, 108, 33, 227, 14, 133, 166, 255, 210, 236, 253, 36, 130, 189, 4, 249, 239, 141, 227, 225, 197, 221, 117, 208, 4, 154, 78, 20, 54, 25, 251, 45, 18, 163, 21, 192, 243, 111, 106, 68, 213, 113, 108, 241, 75, 152, 27, 121, 48, 247, 156, 23, 191, 125, 50, 9, 57, 91, 54, 140, 160, 8, 187, 64, 44, 150, 41, 238, 124, 189, 14, 248, 236, 109, 206, 193, 78, 118, 193, 88, 46, 78, 50, 154, 199, 124, 203, 199, 191, 183, 93, 247, 53, 84, 120, 204, 177, 135, 186, 5, 197, 53, 1, 112, 4, 195, 232, 93, 248, 177, 180, 223, 71, 205, 76, 102, 154, 232, 103, 212, 158, 30, 160, 33, 31, 51, 71, 5, 59, 68, 8, 207, 107, 136, 82, 89, 50, 120, 245, 65, 142, 37, 46, 208, 2, 13, 1, 207, 118, 247, 45, 52, 228, 103, 157, 104, 177, 128, 182, 77, 82, 179, 32, 151, 43, 165, 83, 140, 119, 149, 181, 115, 128, 205, 136, 255, 218, 151, 54, 227, 231, 111, 50, 160, 117, 241, 187, 191, 250, 60, 101, 171, 54, 17, 191, 118, 253, 71, 255, 108, 135, 96, 190, 12, 229, 234, 150, 245, 11, 162, 55, 39, 0, 209, 231, 2, 48, 1, 213, 128, 1, 194, 144, 206, 239, 179, 235, 211, 120, 96, 190, 10, 63, 251, 108, 198, 194, 209, 171, 39, 23, 23, 9, 132, 121, 145, 66, 142, 118, 104, 55, 17, 155, 206, 199, 118, 41, 191, 213, 92, 226, 124, 13, 95, 103, 82, 253, 109, 57, 250, 81, 45, 223, 178, 136, 3, 241, 97, 32, 170, 248, 180, 178, 159, 211, 250, 40, 176, 181, 167, 227, 131, 186, 250, 68, 59, 119, 220, 162, 44, 43, 232, 226, 83, 136, 154, 251, 12, 32, 222, 162, 123, 78, 128, 144, 113, 27, 124, 151, 212, 219, 252, 61, 154, 193, 161, 222, 234, 231, 174, 76, 229, 174, 175, 118, 172, 135, 205, 100, 188, 162, 52, 26, 211, 102, 195, 107, 97, 124, 155, 245, 230, 89, 121, 251, 128, 14, 22, 107, 192, 241, 41, 244, 237, 157, 151, 142, 58, 196, 41, 147, 7, 52, 255, 226, 191, 173, 154, 89, 37, 155, 134, 29, 67, 126, 203, 110, 149, 245, 142, 224, 169, 139, 87, 128, 78, 11, 230, 96, 125, 192, 73, 213, 4, 188, 32, 202, 4, 210, 92, 234, 63, 140, 73, 127, 29, 185, 186, 96, 109, 215, 180, 100, 179, 92, 52, 234, 56, 243, 249, 193, 248, 169, 11, 126, 186, 245, 209, 252, 122, 27, 127, 219, 8, 101, 125, 195, 151, 5, 236, 175, 47, 199, 222, 12, 250, 178, 63, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 87, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 40, 48, 38, 6, 3, 85, 4, 3, 19, 31, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 71, 50, 32, 82, 83, 65, 32, 67, 65, 32, 79, 67, 83, 80, 32, 49, 48],\n                            RawIssuer: [48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [53, 25, 202, 233, 168, 117, 225, 216, 202, 253, 37, 193, 59, 82, 225, 231, 158, 43, 148, 226, 244, 58, 210, 79, 218, 6, 34, 82, 26, 1, 24, 227, 162, 186, 183, 183, 28, 36, 50, 117, 13, 103, 68, 125, 157, 74, 130, 75, 250, 227, 68, 233, 70, 246, 0, 69, 9, 194, 82, 233, 103, 37, 151, 199, 183, 139, 90, 119, 124, 64, 30, 148, 227, 188, 140, 109, 56, 41, 188, 39, 125, 18, 18, 61, 203, 203, 228, 196, 70, 226, 165, 1, 192, 16, 209, 32, 250, 14, 3, 228, 67, 219, 38, 135, 37, 172, 200, 160, 179, 4, 106, 94, 240, 67, 63, 15, 123, 245, 206, 35, 195, 183, 201, 190, 251, 222, 155, 125, 147, 179, 135, 62, 223, 198, 219, 61, 201, 244, 172, 113, 0, 210, 164, 101, 129, 54, 89, 159, 28, 54, 229, 137, 209, 122, 153, 74, 123, 85, 75, 194, 213, 140, 208, 121, 1, 121, 149, 15, 57, 67, 139, 179, 10, 123, 146, 213, 90, 16, 24, 118, 233, 29, 13, 111, 50, 49, 244, 189, 209, 22, 241, 168, 163, 103, 242, 14, 154, 112, 12, 227, 150, 175, 3, 244, 109, 189, 20, 121, 106, 54, 195, 241, 182, 115, 187, 198, 11, 213, 38, 70, 136, 84, 69, 112, 170, 204, 31, 61, 139, 111, 227, 228, 23, 166, 129, 0, 251, 125, 131, 88, 128, 226, 242, 60, 59, 163, 24, 98, 19, 54, 103, 255, 79, 226, 229, 111, 106, 103, 230, 232, 90, 100, 58, 160, 69, 157, 63, 23, 12, 102, 135, 240, 129, 39, 52, 32, 176, 53, 36, 100, 40, 118, 67, 122, 116, 152, 226, 43, 128, 9, 10, 188, 176, 185, 7, 71, 41, 40, 46, 125, 224, 119, 65, 164, 140, 54, 120, 186, 51, 29, 86, 142, 217, 16, 53, 137, 89, 131, 0, 105, 206, 183, 120, 189, 237, 175, 130, 63, 206, 200, 137, 50, 152, 89, 8, 93, 225, 1, 150, 88, 196, 101, 25, 125, 38, 133, 157, 16, 150, 229, 149, 197, 17, 206, 130, 111, 23, 6, 113, 21, 15, 44, 191, 55, 50, 82, 21, 104, 180, 149, 142, 83, 93, 151, 237, 227, 219, 19, 62, 193, 49, 177, 121, 147, 115, 125, 114, 59, 27, 76, 79, 6, 73, 21, 132, 115, 243, 29, 90, 81, 44, 164, 212, 88, 224, 25, 191, 53, 72, 41, 96, 153, 171, 234, 123, 66, 3, 190, 156, 61, 25, 247, 71, 91, 11, 172, 130, 87, 170, 13, 189, 255, 34, 96, 48, 223, 44, 79, 87, 211, 37, 213, 218, 253, 142, 6, 81, 78, 5, 53, 120, 152, 56, 163, 39, 49, 111, 49, 154, 169, 170, 77, 172, 58, 68, 104, 96, 112, 104, 244, 83, 107, 205, 178, 45, 174, 38, 24, 135, 223, 140, 87, 181, 248, 82, 193, 86, 231, 192, 177, 77, 95, 10, 85, 107, 20, 43, 84, 251, 61, 169, 198, 189, 96, 7, 248, 176, 244, 75, 227, 238, 171],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a118ac3670\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [12623528037057212991, 15782976484946675180, 9131881476679932799, 16877507857597638923, 6948446969886104628, 16879364175701064122, 15349600278999781980, 9245340169127444553, 7968545109705329495, 6423711443832766155, 10594494922037898650, 17648935788986680361, 18122500389998752041, 7737602644086511993, 7258855044086458051, 5541026985182791629, 4439073283700877230, 10408130435767131132, 18089869943821258368, 11685763412349126810, 16394152652608075740, 12871239331426121127, 9800942613550528692, 18261315811323797426, 15374412250364208978, 3968124059102095807, 685806708061009512, 7838166407260346135, 16993058175899549691, 15384298262262378419, 3974146097701597185, 13694573362644585378, 13796492994165901152, 13528806942954960401, 3955259362538190321, 13074935763441801879, 2348393719768315797, 11342510827905045171, 130453718204474471, 17672562703128855053, 634844305723830904, 11538538025395239748, 5505257917472087582, 16743812431760803789, 13404336697386206403, 6770939201695035783, 3646446187118575543, 14898275314275200590, 3021489551454039149, 3930692534149328022, 11247669138812385627, 7850138742151524599, 1567520330174748017, 5626181319708709539, 16420623271324550298, 18240848139101269901, 2441811390160163564, 1395980562250961772, 3030591394459285111, 17666945104455592770, 11923213182329091847, 9136006465603430828, 13028421093649262211, 17297287522694218268],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [12560257885759602703, 4278338880794178, 3342336],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS RSA Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS RSA Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS G2 RSA CA OCSP 10\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS G2 RSA CA OCSP 10\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63889675383, loc: nil},\n                            NotAfter: {wall: 0, ext: 64010808183, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 21, 1],\n                                    Critical: false,\n                                    Value: [2, 1, 0],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 4, 1, 311, 20, 2],\n                                    Critical: false,\n                                    Value: [30, 10, 0, 83, 0, 117, 0, 98, 0, 67, 0, 65],\n                                },\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 6, 1, 1, 255, 2, 1, 0],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                                },\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0\\x81\\xa00\\x81\\x9d\\xa0\\x81\\x9a\\xa0\\x81\\x97\\x86Ihttp://www.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\\x86Jhttp://crl2.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 129, 255, 48, 99, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 87, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 105, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 93, 104, 116, 116, 112, 58, 47, 47, 99, 97, 105, 115, 115, 117, 101, 114, 115, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 112, 107, 105, 111, 112, 115, 47, 99, 101, 114, 116, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 37, 50, 48, 84, 76, 83, 37, 50, 48, 82, 83, 65, 37, 50, 48, 82, 111, 111, 116, 37, 50, 48, 71, 50, 37, 50, 48, 45, 37, 50, 48, 120, 115, 105, 103, 110, 46, 99, 114, 116, 48, 45, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 33, 104, 116, 116, 112, 58, 47, 47, 111, 110, 101, 111, 99, 115, 112, 46, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 99, 111, 109, 47, 111, 99, 115, 112],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: 0,\n                            MaxPathLenZero: true,\n                            SubjectKeyId: [208, 76, 131, 209, 142, 113, 46, 61, 52, 177, 255, 155, 121, 44, 142, 116, 212, 226, 120, 6],\n                            AuthorityKeyId: [222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                            OCSPServer: [\n                                \"http://oneocsp.microsoft.com/ocsp\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://www.microsoft.com/pkiops/certs/Microsoft%20TLS%20RSA%20Root%20G2%20-%20xsign.crt\",\n                                \"http://caissuers.microsoft.com/pkiops/certs/Microsoft%20TLS%20RSA%20Root%20G2%20-%20xsign.crt\",\n                            ],\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://www.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                                \"http://crl2.microsoft.com/pkiops/crl/Microsoft%20TLS%20RSA%20Root%20G2.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 5, 137, 48, 130, 4, 113, 160, 3, 2, 1, 2, 2, 16, 11, 12, 107, 44, 70, 105, 23, 176, 71, 115, 198, 71, 212, 175, 192, 200, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 53, 50, 49, 48, 48, 48, 48, 48, 48, 90, 23, 13, 50, 57, 48, 54, 49, 57, 50, 51, 53, 57, 53, 57, 90, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1, 163, 130, 1, 75, 48, 130, 1, 71, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 118, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 106, 48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116, 48, 66, 6, 3, 85, 29, 31, 4, 59, 48, 57, 48, 55, 160, 53, 160, 51, 134, 49, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 51, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 108, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 3, 130, 1, 1, 0, 11, 188, 180, 43, 55, 116, 197, 75, 167, 54, 2, 54, 198, 175, 225, 19, 204, 138, 154, 151, 27, 252, 227, 68, 140, 238, 84, 0, 229, 228, 65, 26, 70, 240, 11, 118, 52, 255, 153, 85, 98, 53, 170, 173, 138, 156, 12, 90, 98, 204, 67, 192, 203, 234, 74, 29, 189, 107, 29, 175, 177, 25, 37, 102, 156, 247, 121, 110, 91, 37, 128, 36, 129, 133, 199, 121, 45, 53, 95, 66, 9, 120, 237, 159, 226, 101, 213, 196, 32, 80, 130, 16, 143, 106, 58, 237, 24, 142, 72, 137, 171, 166, 9, 248, 225, 128, 252, 6, 117, 22, 39, 37, 90, 143, 184, 58, 112, 108, 24, 71, 240, 99, 244, 83, 14, 217, 21, 53, 32, 243, 145, 61, 19, 1, 160, 226, 250, 36, 8, 69, 86, 119, 209, 103, 169, 49, 66, 208, 87, 255, 26, 38, 190, 197, 67, 216, 87, 169, 99, 255, 175, 243, 58, 216, 246, 123, 75, 16, 162, 151, 161, 207, 102, 190, 103, 103, 9, 249, 181, 76, 231, 171, 25, 43, 191, 34, 61, 125, 222, 198, 247, 239, 99, 56, 44, 221, 86, 196, 249, 9, 82, 218, 223, 227, 136, 111, 213, 109, 107, 247, 8, 65, 47, 142, 62, 134, 49, 159, 241, 124, 74, 245, 214, 97, 164, 70, 70, 40, 239, 87, 186, 176, 111, 146, 216, 210, 119, 37, 237, 123, 166, 41, 206, 59, 114, 121, 17, 212, 224, 167, 84, 109, 53, 43, 164, 122],\n                            RawTBSCertificate: [48, 130, 4, 113, 160, 3, 2, 1, 2, 2, 16, 11, 12, 107, 44, 70, 105, 23, 176, 71, 115, 198, 71, 212, 175, 192, 200, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 12, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 50, 53, 48, 53, 50, 49, 48, 48, 48, 48, 48, 48, 90, 23, 13, 50, 57, 48, 54, 49, 57, 50, 51, 53, 57, 53, 57, 90, 48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1, 163, 130, 1, 75, 48, 130, 1, 71, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118, 48, 31, 6, 3, 85, 29, 35, 4, 24, 48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 19, 6, 3, 85, 29, 37, 4, 12, 48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 118, 6, 8, 43, 6, 1, 5, 5, 7, 1, 1, 4, 106, 48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116, 48, 66, 6, 3, 85, 29, 31, 4, 59, 48, 57, 48, 55, 160, 53, 160, 51, 134, 49, 104, 116, 116, 112, 58, 47, 47, 99, 114, 108, 51, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 108, 48, 19, 6, 3, 85, 29, 32, 4, 12, 48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                            RawSubjectPublicKeyInfo: [48, 130, 2, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 2, 15, 0, 48, 130, 2, 10, 2, 130, 2, 1, 0, 223, 234, 139, 159, 71, 225, 40, 16, 113, 175, 65, 214, 25, 219, 146, 87, 222, 107, 171, 230, 46, 222, 173, 58, 96, 131, 183, 19, 152, 172, 91, 185, 50, 11, 123, 235, 159, 114, 39, 107, 93, 64, 171, 190, 24, 6, 147, 178, 13, 210, 18, 237, 184, 116, 245, 127, 47, 234, 168, 23, 148, 246, 51, 22, 87, 138, 222, 32, 124, 33, 221, 183, 218, 172, 222, 78, 100, 176, 90, 205, 75, 117, 243, 239, 2, 100, 77, 237, 82, 218, 163, 143, 4, 232, 145, 248, 215, 251, 63, 25, 253, 78, 69, 57, 94, 232, 41, 195, 214, 242, 103, 175, 179, 0, 148, 193, 141, 225, 240, 74, 24, 152, 57, 250, 192, 9, 132, 51, 65, 5, 3, 5, 171, 244, 117, 225, 83, 165, 33, 146, 92, 68, 76, 230, 96, 87, 164, 144, 2, 151, 199, 213, 216, 147, 4, 50, 74, 237, 133, 81, 237, 214, 197, 186, 160, 63, 54, 138, 47, 111, 76, 57, 168, 4, 105, 34, 132, 59, 117, 20, 247, 217, 62, 180, 169, 208, 7, 65, 56, 129, 126, 70, 72, 29, 232, 25, 230, 112, 184, 160, 170, 24, 221, 114, 191, 126, 125, 181, 108, 78, 25, 112, 17, 132, 4, 19, 129, 212, 98, 253, 156, 107, 241, 28, 221, 255, 236, 167, 224, 230, 199, 141, 49, 75, 191, 242, 107, 51, 204, 209, 140, 182, 254, 177, 27, 163, 74, 168, 95, 14, 127, 49, 118, 56, 63, 7, 71, 108, 1, 168, 172, 165, 20, 218, 130, 176, 95, 31, 108, 188, 137, 26, 192, 18, 42, 203, 123, 151, 31, 40, 46, 49, 22, 138, 83, 76, 117, 102, 249, 125, 1, 214, 194, 96, 170, 101, 143, 49, 158, 114, 203, 204, 155, 45, 229, 68, 25, 147, 87, 171, 110, 142, 194, 217, 46, 101, 176, 114, 45, 231, 161, 195, 161, 126, 73, 29, 89, 103, 214, 9, 212, 24, 18, 109, 192, 149, 69, 120, 201, 241, 96, 106, 228, 246, 238, 6, 36, 59, 115, 74, 90, 17, 194, 62, 151, 97, 192, 120, 238, 239, 19, 167, 48, 226, 74, 56, 45, 238, 88, 23, 19, 32, 183, 54, 42, 223, 11, 50, 180, 194, 56, 177, 145, 221, 18, 176, 69, 182, 124, 192, 177, 29, 98, 161, 157, 221, 113, 18, 21, 150, 1, 157, 10, 9, 245, 177, 221, 179, 217, 224, 56, 66, 52, 246, 105, 182, 195, 175, 22, 115, 254, 212, 67, 248, 67, 228, 88, 249, 79, 179, 243, 236, 214, 61, 148, 58, 64, 56, 21, 171, 12, 134, 208, 100, 206, 26, 99, 232, 132, 148, 85, 9, 90, 182, 4, 182, 209, 62, 154, 21, 199, 100, 17, 128, 208, 27, 130, 14, 77, 255, 69, 158, 178, 101, 7, 37, 31, 219, 154, 200, 198, 236, 214, 91, 133, 80, 4, 159, 28, 203, 93, 32, 130, 103, 158, 121, 226, 223, 232, 242, 232, 111, 196, 32, 114, 48, 134, 229, 157, 117, 39, 53, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 81, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 30, 48, 28, 6, 3, 85, 4, 10, 19, 21, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 67, 111, 114, 112, 111, 114, 97, 116, 105, 111, 110, 49, 34, 48, 32, 6, 3, 85, 4, 3, 19, 25, 77, 105, 99, 114, 111, 115, 111, 102, 116, 32, 84, 76, 83, 32, 82, 83, 65, 32, 82, 111, 111, 116, 32, 71, 50],\n                            RawIssuer: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [11, 188, 180, 43, 55, 116, 197, 75, 167, 54, 2, 54, 198, 175, 225, 19, 204, 138, 154, 151, 27, 252, 227, 68, 140, 238, 84, 0, 229, 228, 65, 26, 70, 240, 11, 118, 52, 255, 153, 85, 98, 53, 170, 173, 138, 156, 12, 90, 98, 204, 67, 192, 203, 234, 74, 29, 189, 107, 29, 175, 177, 25, 37, 102, 156, 247, 121, 110, 91, 37, 128, 36, 129, 133, 199, 121, 45, 53, 95, 66, 9, 120, 237, 159, 226, 101, 213, 196, 32, 80, 130, 16, 143, 106, 58, 237, 24, 142, 72, 137, 171, 166, 9, 248, 225, 128, 252, 6, 117, 22, 39, 37, 90, 143, 184, 58, 112, 108, 24, 71, 240, 99, 244, 83, 14, 217, 21, 53, 32, 243, 145, 61, 19, 1, 160, 226, 250, 36, 8, 69, 86, 119, 209, 103, 169, 49, 66, 208, 87, 255, 26, 38, 190, 197, 67, 216, 87, 169, 99, 255, 175, 243, 58, 216, 246, 123, 75, 16, 162, 151, 161, 207, 102, 190, 103, 103, 9, 249, 181, 76, 231, 171, 25, 43, 191, 34, 61, 125, 222, 198, 247, 239, 99, 56, 44, 221, 86, 196, 249, 9, 82, 218, 223, 227, 136, 111, 213, 109, 107, 247, 8, 65, 47, 142, 62, 134, 49, 159, 241, 124, 74, 245, 214, 97, 164, 70, 70, 40, 239, 87, 186, 176, 111, 146, 216, 210, 119, 37, 237, 123, 166, 41, 206, 59, 114, 121, 17, 212, 224, 167, 84, 109, 53, 43, 164, 122],\n                            SignatureAlgorithm: 5,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a118ac37b0\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [8228224839953229621, 16348041302357558304, 2074854447542410873, 14334067402580427935, 12854688567428618952, 14995722382209926558, 15077658020813214080, 9553354208224609462, 902637906707506152, 15437658075602359723, 4892132923038495724, 14100513645772686328, 15699610158021110198, 116260203274558899, 7107135261377697174, 1346652938663145757, 806906029520556509, 6347563231035665119, 1416154356297248238, 13996791138010853103, 17151436068643035665, 5005973024364225782, 15423091598760460437, 11656337824329587047, 14040304327591079399, 16520357453723561614, 10318202437165161261, 17977527309428632165, 3328466382962324838, 13840171658303184680, 9417131409404627226, 5146490298140267738, 6849551933690625799, 10139571647697603240, 3552143779013315793, 15996764607220205453, 9355211167345930524, 7804203173400544275, 12256789870797553077, 5196564742993393824, 12236288164267916870, 9528338169550421684, 3418034452713204002, 17138102837348087434, 15605821747820856657, 6942198289040656341, 6027260588802198758, 4685154310243513825, 1772230202496746547, 12898472891819094090, 6838761955263670191, 15563102318165378361, 5970264090691015160, 5437520332982603245, 15757213624540617421, 6308098459188714935, 3452756883569783574, 995879279813522815, 6719559476972000178, 3606112178562803563, 6954603544383871929, 16027092704045215034, 8191838635022848599, 16134862131230222352],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [5148676810827874504, 796129072041760688],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\n                                    \"Microsoft Corporation\",\n                                ],\n                                OrganizationalUnit: nil,\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"Microsoft TLS RSA Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft Corporation\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"Microsoft TLS RSA Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63883382400, loc: nil},\n                            NotAfter: {wall: 0, ext: 64012204799, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 3, 1, 1, 255],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                                },\n                                {\n                                    Id: [2, 5, 29, 35],\n                                    Critical: false,\n                                    Value: [48, 22, 128, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [2, 5, 29, 37],\n                                    Critical: false,\n                                    Value: [48, 10, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],\n                                },\n                                {\n                                    Id: [1, 3, 6, 1, 5, 5, 7, 1, 1],\n                                    Critical: false,\n                                    Value: [48, 104, 48, 36, 6, 8, 43, 6, 1, 5, 5, 7, 48, 1, 134, 24, 104, 116, 116, 112, 58, 47, 47, 111, 99, 115, 112, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 48, 64, 6, 8, 43, 6, 1, 5, 5, 7, 48, 2, 134, 52, 104, 116, 116, 112, 58, 47, 47, 99, 97, 99, 101, 114, 116, 115, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 47, 68, 105, 103, 105, 67, 101, 114, 116, 71, 108, 111, 98, 97, 108, 82, 111, 111, 116, 71, 50, 46, 99, 114, 116],\n                                },\n                                {\n                                    Id: [2, 5, 29, 31],\n                                    Critical: false,\n                                    Value: \"0907\\xa05\\xa03\\x861http://crl3.digicert.com/DigiCertGlobalRootG2.crl\",\n                                },\n                                {\n                                    Id: [2, 5, 29, 32],\n                                    Critical: false,\n                                    Value: [48, 10, 48, 8, 6, 6, 103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: [1],\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [222, 145, 134, 72, 183, 161, 49, 89, 49, 241, 75, 95, 7, 169, 220, 136, 121, 218, 168, 118],\n                            AuthorityKeyId: [78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            OCSPServer: [\n                                \"http://ocsp.digicert.com\",\n                            ],\n                            IssuingCertificateURL: [\n                                \"http://cacerts.digicert.com/DigiCertGlobalRootG2.crt\",\n                            ],\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: [\n                                \"http://crl3.digicert.com/DigiCertGlobalRootG2.crl\",\n                            ],\n                            PolicyIdentifiers: [\n                                [2, 23, 140, 1, 2, 2],\n                            ],\n                            Policies: [\n                                {\n                                    der: [103, 129, 12, 1, 2, 2],\n                                },\n                            ],\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                        {\n                            Raw: [48, 130, 3, 142, 48, 130, 2, 118, 160, 3, 2, 1, 2, 2, 16, 3, 58, 241, 230, 167, 17, 169, 160, 187, 40, 100, 177, 29, 9, 250, 229, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 49, 51, 48, 56, 48, 49, 49, 50, 48, 48, 48, 48, 90, 23, 13, 51, 56, 48, 49, 49, 53, 49, 50, 48, 48, 48, 48, 90, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1, 163, 66, 48, 64, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 96, 103, 40, 148, 111, 14, 72, 99, 235, 49, 221, 234, 103, 24, 213, 137, 125, 60, 197, 139, 74, 127, 233, 190, 219, 43, 23, 223, 176, 95, 115, 119, 42, 50, 19, 57, 129, 103, 66, 132, 35, 242, 69, 103, 53, 236, 136, 191, 248, 143, 176, 97, 12, 52, 164, 174, 32, 76, 132, 198, 219, 248, 53, 225, 118, 217, 223, 166, 66, 187, 199, 68, 8, 134, 127, 54, 116, 36, 90, 218, 108, 13, 20, 89, 53, 189, 242, 73, 221, 182, 31, 201, 179, 13, 71, 42, 61, 153, 47, 187, 92, 187, 181, 212, 32, 225, 153, 95, 83, 70, 21, 219, 104, 155, 240, 243, 48, 213, 62, 49, 226, 141, 132, 158, 227, 138, 218, 218, 150, 62, 53, 19, 165, 95, 240, 249, 112, 80, 112, 71, 65, 17, 87, 25, 78, 192, 143, 174, 6, 196, 149, 19, 23, 47, 27, 37, 159, 117, 242, 177, 142, 153, 161, 111, 19, 177, 65, 113, 254, 136, 42, 200, 79, 16, 32, 85, 215, 243, 20, 69, 229, 224, 68, 244, 234, 135, 149, 50, 147, 14, 254, 83, 70, 250, 44, 157, 255, 139, 34, 185, 75, 217, 9, 69, 164, 222, 164, 184, 154, 88, 221, 27, 125, 82, 159, 142, 89, 67, 136, 129, 164, 158, 38, 213, 111, 173, 221, 13, 198, 55, 125, 237, 3, 146, 27, 229, 119, 95, 118, 238, 60, 141, 196, 93, 86, 91, 162, 217, 102, 110, 179, 53, 55, 229, 50, 182],\n                            RawTBSCertificate: [48, 130, 2, 118, 160, 3, 2, 1, 2, 2, 16, 3, 58, 241, 230, 167, 17, 169, 160, 187, 40, 100, 177, 29, 9, 250, 229, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 30, 23, 13, 49, 51, 48, 56, 48, 49, 49, 50, 48, 48, 48, 48, 90, 23, 13, 51, 56, 48, 49, 49, 53, 49, 50, 48, 48, 48, 48, 90, 48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1, 163, 66, 48, 64, 48, 15, 6, 3, 85, 29, 19, 1, 1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 187, 55, 205, 52, 220, 123, 107, 201, 178, 104, 144, 173, 74, 117, 255, 70, 186, 33, 10, 8, 141, 245, 25, 84, 201, 251, 136, 219, 243, 174, 242, 58, 137, 145, 60, 122, 230, 171, 6, 26, 107, 207, 172, 45, 232, 94, 9, 36, 68, 186, 98, 154, 126, 214, 163, 168, 126, 224, 84, 117, 32, 5, 172, 80, 183, 156, 99, 26, 108, 48, 220, 218, 31, 25, 177, 215, 30, 222, 253, 215, 224, 203, 148, 131, 55, 174, 236, 31, 67, 78, 221, 123, 44, 210, 189, 46, 165, 47, 228, 169, 184, 173, 58, 212, 153, 164, 182, 37, 233, 155, 107, 0, 96, 146, 96, 255, 79, 33, 73, 24, 247, 103, 144, 171, 97, 6, 156, 143, 242, 186, 233, 180, 233, 146, 50, 107, 181, 243, 87, 232, 93, 27, 205, 140, 29, 171, 149, 4, 149, 73, 243, 53, 45, 150, 227, 73, 109, 221, 119, 227, 251, 73, 75, 180, 172, 85, 7, 169, 143, 149, 179, 180, 35, 187, 76, 109, 69, 240, 246, 169, 178, 149, 48, 180, 253, 76, 85, 140, 39, 74, 87, 20, 124, 130, 157, 205, 115, 146, 211, 22, 74, 6, 12, 140, 80, 209, 143, 30, 9, 190, 23, 161, 230, 33, 202, 253, 131, 229, 16, 188, 131, 165, 10, 196, 103, 40, 246, 115, 20, 20, 61, 70, 118, 195, 135, 20, 137, 33, 52, 77, 175, 15, 69, 12, 166, 73, 161, 186, 187, 156, 197, 177, 51, 131, 41, 133, 2, 3, 1, 0, 1],\n                            RawSubject: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            RawIssuer: [48, 97, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2, 85, 83, 49, 21, 48, 19, 6, 3, 85, 4, 10, 19, 12, 68, 105, 103, 105, 67, 101, 114, 116, 32, 73, 110, 99, 49, 25, 48, 23, 6, 3, 85, 4, 11, 19, 16, 119, 119, 119, 46, 100, 105, 103, 105, 99, 101, 114, 116, 46, 99, 111, 109, 49, 32, 48, 30, 6, 3, 85, 4, 3, 19, 23, 68, 105, 103, 105, 67, 101, 114, 116, 32, 71, 108, 111, 98, 97, 108, 32, 82, 111, 111, 116, 32, 71, 50],\n                            Signature: [96, 103, 40, 148, 111, 14, 72, 99, 235, 49, 221, 234, 103, 24, 213, 137, 125, 60, 197, 139, 74, 127, 233, 190, 219, 43, 23, 223, 176, 95, 115, 119, 42, 50, 19, 57, 129, 103, 66, 132, 35, 242, 69, 103, 53, 236, 136, 191, 248, 143, 176, 97, 12, 52, 164, 174, 32, 76, 132, 198, 219, 248, 53, 225, 118, 217, 223, 166, 66, 187, 199, 68, 8, 134, 127, 54, 116, 36, 90, 218, 108, 13, 20, 89, 53, 189, 242, 73, 221, 182, 31, 201, 179, 13, 71, 42, 61, 153, 47, 187, 92, 187, 181, 212, 32, 225, 153, 95, 83, 70, 21, 219, 104, 155, 240, 243, 48, 213, 62, 49, 226, 141, 132, 158, 227, 138, 218, 218, 150, 62, 53, 19, 165, 95, 240, 249, 112, 80, 112, 71, 65, 17, 87, 25, 78, 192, 143, 174, 6, 196, 149, 19, 23, 47, 27, 37, 159, 117, 242, 177, 142, 153, 161, 111, 19, 177, 65, 113, 254, 136, 42, 200, 79, 16, 32, 85, 215, 243, 20, 69, 229, 224, 68, 244, 234, 135, 149, 50, 147, 14, 254, 83, 70, 250, 44, 157, 255, 139, 34, 185, 75, 217, 9, 69, 164, 222, 164, 184, 154, 88, 221, 27, 125, 82, 159, 142, 89, 67, 136, 129, 164, 158, 38, 213, 111, 173, 221, 13, 198, 55, 125, 237, 3, 146, 27, 229, 119, 95, 118, 238, 60, 141, 196, 93, 86, 91, 162, 217, 102, 110, 179, 53, 55, 229, 50, 182],\n                            SignatureAlgorithm: 4,\n                            PublicKeyAlgorithm: 1,\n                            PublicKey: \u003c*rsa.PublicKey | 0x36a11782c840\u003e{\n                                N: {\n                                    neg: false,\n                                    abs: [13518897546323503493, 12614377001919816122, 8557832239203824717, 7433462159093546310, 9504020990483827396, 702024576034523901, 5333964605457796894, 8971906913408307990, 18252057450268743444, 5039799191733612724, 10346373305228545133, 18107087016177239977, 3285062882728703971, 2137966295034491701, 13110919595711778188, 17490549066068800107, 17827376715711028367, 6958731023901346072, 11071174057932843776, 11902983757762476756, 4850057369827327278, 16198203776063958047, 2241017827087547863, 13230558770443115738, 9142400105587256400, 4952379155939763112, 7768617195409377572, 9912770753372816922, 14554377099045433914, 13412012197143714132, 12855684210282069830, 13490476835732679625],\n                                },\n                                E: 65537,\n                            },\n                            Version: 3,\n                            SerialNumber: {\n                                neg: false,\n                                abs: [13486139796020132581, 232764303710726560],\n                            },\n                            Issuer: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            Subject: {\n                                Country: [\"US\"],\n                                Organization: [\"DigiCert Inc\"],\n                                OrganizationalUnit: [\"www.digicert.com\"],\n                                Locality: nil,\n                                Province: nil,\n                                StreetAddress: nil,\n                                PostalCode: nil,\n                                SerialNumber: \"\",\n                                CommonName: \"DigiCert Global Root G2\",\n                                Names: [\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"US\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Inc\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"www.digicert.com\",\n                                    },\n                                    {\n                                        Type: [..., ..., ..., ...],\n                                        Value: \u003cstring\u003e\"DigiCert Global Root G2\",\n                                    },\n                                ],\n                                ExtraNames: nil,\n                            },\n                            NotBefore: {wall: 0, ext: 63510955200, loc: nil},\n                            NotAfter: {wall: 0, ext: 64282766400, loc: nil},\n                            KeyUsage: 97,\n                            Extensions: [\n                                {\n                                    Id: [2, 5, 29, 19],\n                                    Critical: true,\n                                    Value: [48, 3, 1, 1, 255],\n                                },\n                                {\n                                    Id: [2, 5, 29, 15],\n                                    Critical: true,\n                                    Value: [3, 2, 1, 134],\n                                },\n                                {\n                                    Id: [2, 5, 29, 14],\n                                    Critical: false,\n                                    Value: [4, 20, 78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                                },\n                            ],\n                            ExtraExtensions: nil,\n                            UnhandledCriticalExtensions: nil,\n                            ExtKeyUsage: nil,\n                            UnknownExtKeyUsage: nil,\n                            BasicConstraintsValid: true,\n                            IsCA: true,\n                            MaxPathLen: -1,\n                            MaxPathLenZero: false,\n                            SubjectKeyId: [78, 34, 84, 32, 24, 149, 230, 227, 110, 230, 15, 250, 250, 185, 18, 237, 6, 23, 143, 57],\n                            AuthorityKeyId: nil,\n                            OCSPServer: nil,\n                            IssuingCertificateURL: nil,\n                            DNSNames: nil,\n                            EmailAddresses: nil,\n                            IPAddresses: nil,\n                            URIs: nil,\n                            PermittedDNSDomainsCritical: false,\n                            PermittedDNSDomains: nil,\n                            ExcludedDNSDomains: nil,\n                            PermittedIPRanges: nil,\n                            ExcludedIPRanges: nil,\n                            PermittedEmailAddresses: nil,\n                            ExcludedEmailAddresses: nil,\n                            PermittedURIDomains: nil,\n                            ExcludedURIDomains: nil,\n                            CRLDistributionPoints: nil,\n                            PolicyIdentifiers: nil,\n                            Policies: nil,\n                            InhibitAnyPolicy: 0,\n                            InhibitAnyPolicyZero: false,\n                            InhibitPolicyMapping: 0,\n                            InhibitPolicyMappingZero: false,\n                            RequireExplicitPolicy: 0,\n                            RequireExplicitPolicyZero: false,\n                            PolicyMappings: nil,\n                        },\n                    ],\n                    Err: \u003cx509.UnknownAuthorityError\u003e{\n                        Cert: 0x36a1182f5208 (cyclic reference),\n                        hintErr: nil,\n                        hintCert: nil,\n                    },\n                },\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114175122356047872","occurred_at":"2026-10-25T02:00:03Z","signature_id":"e0aaaa221e9f5109d3c4f1d73b9d2e58ae03f18b791fff7e6fea11c31f369189","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b5a5961714f637388dd3e0c812afef07486cc85e1d072581febf21aa97e03c51","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T02:00:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:493]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1a2d0a2b0a40\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:493]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1a2d0a2b0a40\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2113450457698406400","occurred_at":"2026-10-23T02:00:29Z","signature_id":"dafe564804491cb1b5abebe84a25ff5ad04e28db49131d7f1ce224ae40b9a4a9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c349dea01a43efbfe8bada3f00e6006e559aa06c48ac45e43b919b7539b25770","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-19"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-23T02:00:29Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:510]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3b09d8d2ba10\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115375462514429952","occurred_at":"2026-10-28T09:29:46Z","signature_id":"bfb9b703f5241dd736b9902336860998ffe6bdf4ba01f9a3a8b4d81786acbd9e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x8979cd42dc0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-d4wglkkdjhfj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"name\": \"4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-28T03:01:21.21908488Z\",\n      \"endTime\": \"2026-10-28T03:20:21.275752007Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p0i5l3o3i9z8d7r-np-deletable-95t5c-lvd52: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4691f9f8-c66b-49b4-9a08-68ce0d3e72a6\n            correlation ID: 48b19522-cae3-4702-8aae-982e9a4042a0\n            client request ID: unavailable"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2115262480291729408","occurred_at":"2026-10-28T02:00:49Z","signature_id":"a130358adaef86ab6bdbcc865713e8b92da4c632acc5d4fbbe39da19d0be4f65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3e7244a85da8bd67de02bd0b8b84b756179c64ce2823e309e63b548314c55c8e","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management","failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-21..2026-10-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"unknown","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","signature_id":"b9c3c5012ff77c3681e350e06c44d7277c66955d064a006d326eb01c4d066632","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}