{"meta":{"start_date":"2026-10-19","end_date":"2026-10-19","start_at":"2026-10-19T00:00:00Z","end_at":"2026-10-20T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T07:01:25Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":92,"failed_runs":75,"raw_occurrences":150,"matched_occurrences":143,"runs_affected":68},"rows":[{"environment":"dev","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":21,"runs_affected":21,"failed_runs":21,"run_impact_percent":22.82608695652174,"anchor_occurrences":21,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,21],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":21,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":21}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112230579016568832","occurred_at":"2026-10-19T17:13:05Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112229674405859328","occurred_at":"2026-10-19T17:09:32Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222204212023296","occurred_at":"2026-10-19T16:39:51Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222016026185728","occurred_at":"2026-10-19T16:39:06Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7604,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221900917706752","occurred_at":"2026-10-19T16:38:38Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7556,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221482724626432","occurred_at":"2026-10-19T16:36:57Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112212933982294016","occurred_at":"2026-10-19T16:02:58Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7613,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112211956654936064","occurred_at":"2026-10-19T15:59:08Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7601,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7132,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205019058016256","occurred_at":"2026-10-19T15:31:34Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7609,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199925969195008","occurred_at":"2026-10-19T15:11:19Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7597,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112193616469299200","occurred_at":"2026-10-19T14:46:15Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112186650028150784","occurred_at":"2026-10-19T14:18:34Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7556,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112184932984950784","occurred_at":"2026-10-19T14:11:45Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112183526613848064","occurred_at":"2026-10-19T14:06:05Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112177636460466176","occurred_at":"2026-10-19T13:42:45Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":7220,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173510980800512","occurred_at":"2026-10-19T13:26:21Z","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","pr_number":6718,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":14.130434782608695,"anchor_occurrences":13,"after_last_push_count":35,"trend_counts":[6,4,10,9,1,2,13],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":128,"prior_last_seen_at":"2026-10-18T21:09:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":13}],"full_error_samples":["alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:44:19Z\n Ended: 2026-10-19T22:50:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4417280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti4iojordd6hm0nnh191a3fmg3e1aub-f4w6r1h1a4m1v8n\", pod=\"openshift-apiserver-7dc759866b-hr75b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti4iojordd6hm0nnh191a3fmg3e1aub-f4w6r1h1a4m1v8n/openshift-apiserver-7dc759866b-hr75b has been in a non-ready state for longer than 5 minutes.","alert fired 10 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-rbqnk\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-rbqnk has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-vr6v5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-vr6v5 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-9vsw9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-9vsw9 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3pc5270b8h60pst14u078uaplvhlq-q0m7n1d5p4o1u4j\", pod=\"azure-disk-csi-driver-controller-65f5bf78b-s9gq4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3pc5270b8h60pst14u078uaplvhlq-q0m7n1d5p4o1u4j/azure-disk-csi-driver-controller-65f5bf78b-s9gq4 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-sb8lp\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-sb8lp has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-bjj96\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-bjj96 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-w4b2v\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-w4b2v has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-8qtrk\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-8qtrk has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-mkmmq\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-mkmmq has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-rvxrh\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-rvxrh has been in a non-ready state for longer than 5 minutes.","alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T20:56:48Z\n Ended: 2026-10-19T21:03:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7537536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti35rfutnepdsfn0vfq6aa39v1svufp-f2i3r5n2t0v2v3v\", pod=\"router-6dbb4f8499-ccrn9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti35rfutnepdsfn0vfq6aa39v1svufp-f2i3r5n2t0v2v3v/router-6dbb4f8499-ccrn9 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112293841544417280","occurred_at":"2026-10-19T21:24:30Z","signature_id":"4ece708f0a144a12dcdf372fed7f3d095b244cb8f59c1436a40f58700cd7033b","pr_number":7547,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112281126293737472","occurred_at":"2026-10-19T20:33:59Z","signature_id":"1584cff6139bca6b6622e28aa8cc37a138dc43ddd12fb3a8e76e579f7fdcf427","pr_number":7612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112274862377537536","occurred_at":"2026-10-19T20:09:05Z","signature_id":"dfee0196c4ef11f2d05adeebeabed98c6ade9a65dc79ffe19ad01f1c11c7117f","pr_number":7604,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112258228648677376","occurred_at":"2026-10-19T19:03:00Z","signature_id":"08981d454c0029f1ada7b11e8d8104fb622c4bfeb70379b2cd58fc966c5e5ded","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112244635597279232","occurred_at":"2026-10-19T18:08:59Z","signature_id":"49f892d173e1abb2197dc4498b86e904e0237ccafbf4467bc06a54735fd7ea43","pr_number":7609,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112230579016568832","occurred_at":"2026-10-19T17:13:05Z","signature_id":"736153835c9cc9903671d0fd4f58a966daad1408a64970eb05dba83ccede2b68","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","signature_id":"8e1d1e2137e01eb5a249e9a1bb923b9498a07e23c1b6d569a52d3f1090035430","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205019058016256","occurred_at":"2026-10-19T15:31:34Z","signature_id":"85e0534a15b4dde124a5fdccdabf16fc4b85dce9f7c3b17f9ae34aed46461e4c","pr_number":7609,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","signature_id":"6c8c0b3e9ef7152e716e2cd26aff799256e0121f5ab204a6b434f09e208def1d","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112186650028150784","occurred_at":"2026-10-19T14:18:34Z","signature_id":"fba563d3cf21c439e21e52fbff6e8826a48f4c80cb44b801ea66fc423a3d8251","pr_number":7556,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112177636460466176","occurred_at":"2026-10-19T13:42:45Z","signature_id":"2e81a638699deccff676fadc9bea465d89c45dd7ef0f11af7ee538a64f2f6d78","pr_number":7220,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","signature_id":"ced778a05ef7140f9fbc99e57d92ba12b8ad04f552e5d2603281fc04299f4213","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"5f97d4e148771e1613ddbaeca969a031101167ea329462e75b57cd1d7557c9ee","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":9.782608695652174,"anchor_occurrences":9,"after_last_push_count":0,"trend_counts":[0,0,2,2,0,0,9],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-16T22:50:55Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","occurrences":5},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","occurrences":4}],"full_error_samples":["time=2026-10-20T00:14:03.097Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-20T00:14:03.476Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-20T00:14:03.476Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:14:03.887Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-20T00:14:04.156Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-20T00:14:04.156Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:14:04.247Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:15:01.574Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-20T00:15:01.574Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-19T20:14:27.366Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T20:14:27.872Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T20:14:27.872Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:14:28.027Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-19T20:14:28.232Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-19T20:14:28.232Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:14:28.292Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:15:24.446Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T20:15:24.446Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-19T19:34:54.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T19:34:54.990Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T19:34:54.990Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:34:55.299Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-19T19:34:55.456Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-19T19:34:55.456Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:34:55.507Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:35:45.081Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T19:35:45.081Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112322496756715520","occurred_at":"2026-10-19T23:18:22Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112270689380601856","occurred_at":"2026-10-19T19:52:24Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260172159127552","occurred_at":"2026-10-19T19:10:43Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112240367163150336","occurred_at":"2026-10-19T17:52:01Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112228565553516544","occurred_at":"2026-10-19T17:05:07Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112213551379648512","occurred_at":"2026-10-19T16:05:17Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112207281033580544","occurred_at":"2026-10-19T15:40:33Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112195721783414784","occurred_at":"2026-10-19T14:54:36Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112075402955460608","occurred_at":"2026-10-19T06:56:31Z","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":9.782608695652174,"anchor_occurrences":9,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,9],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":9}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x3eb77839e738\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x3eb77839e738\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x62872c04378\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x62872c04378\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x25effd432cd8\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x25effd432cd8\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"f980adf8603fc169c4ac17be1c7ca63fdf5ff2e144f6e05dfce3213acbfd3546","pr_number":7606,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173242356600832","occurred_at":"2026-10-19T13:25:17Z","signature_id":"c187448bc769289cb30ffae95a4dd66c338ebeb7dbda51deeb2382c564529a67","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112168271166836736","occurred_at":"2026-10-19T13:05:30Z","signature_id":"ca2d2cbad258d0e0b68f66bc68f3332bc697e149cae9b0a98284c3671d947cb5","pr_number":7604,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","signature_id":"7729be770e990dbf343ecf49871e27b862504692b1c6c1526fc07e8d52d20aa5","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112163579019923456","occurred_at":"2026-10-19T12:46:53Z","signature_id":"46e4798aef12db46f28e39faecd964135fd6e1d80259c6a8305854e5da705157","pr_number":7497,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"571bfa44b18692af70f32970a9dbbe3eb0277ef6a3941e907e7523d8a253cfb0","pr_number":7603,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112155408402485248","occurred_at":"2026-10-19T12:14:25Z","signature_id":"70c1e97b6ee68a8af51a0d1a084c8bb6d1520a0dc4d7df34c5a4b515db5df67b","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112154781085601792","occurred_at":"2026-10-19T12:11:56Z","signature_id":"ae04773ede4b41e57386dc7ca80e3ac46afe158f711be0190a4f51cb38c2dfb1","pr_number":7597,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112144183836807168","occurred_at":"2026-10-19T11:29:49Z","signature_id":"9b37230de007f4358ce6b69050e67aa0a03a63570db97f0b34fc803d04c59567","pr_number":7065,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":6.521739130434782,"anchor_occurrences":6,"after_last_push_count":76,"trend_counts":[9,12,11,6,0,0,6],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":239,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-20T00:03:40Z\n Ended: 2026-10-20T00:18:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0435200-svc\", component=\"backend\", name=\"ingresscertificate\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue ingresscertificate has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:38:15Z\n Ended: 2026-10-19T16:53:16Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5762560-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T15:19:57Z\n Ended: 2026-10-19T15:33:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4950784-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T15:24:05Z\n Ended: 2026-10-19T15:33:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4950784-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","signature_id":"f0c211f0475f4d8abc5d3fd2c2049832c9ac8b9c7ee9259b6cfe55ec42e3266f","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7132,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112184932984950784","occurred_at":"2026-10-19T14:11:45Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112094774411399168","occurred_at":"2026-10-19T08:13:10Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7599,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.434782608695652,"anchor_occurrences":5,"after_last_push_count":13,"trend_counts":[6,3,1,0,0,0,5],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":41,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-d9xldnlstk6j/cluster-candidate-5-0-tlxgzk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c66751baf40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tlxgzk: failed waiting for cluster=\"cluster-candidate-5-0-tlxgzk\" in resourcegroup=\"rg-candidate-5-0-d9xldnlstk6j\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\",\n \"name\": \"6dc91de3-05e9-401b-9139-c466ece83065\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:04:14.113898084Z\",\n \"endTime\": \"2026-10-19T22:33:23.409451279Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n correlation ID: 2a2d3ba7-babe-4fe8-ae4e-5435562f5656\n client request ID: a7fdb086-12e2-40d2-b194-107f4afa279d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-d9xldnlstk6j/cluster-candidate-5-0-tlxgzk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c66751baf40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tlxgzk: failed waiting for cluster=\"cluster-candidate-5-0-tlxgzk\" in resourcegroup=\"rg-candidate-5-0-d9xldnlstk6j\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\",\n \"name\": \"6dc91de3-05e9-401b-9139-c466ece83065\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:04:14.113898084Z\",\n \"endTime\": \"2026-10-19T22:33:23.409451279Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n correlation ID: 2a2d3ba7-babe-4fe8-ae4e-5435562f5656\n client request ID: a7fdb086-12e2-40d2-b194-107f4afa279d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xdbc40b14b80\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9ndqj45cmqg2\" to finish creating: GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"name\": \"a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:15:40.641153712Z\",\n \"endTime\": \"2026-10-19T18:44:47.329454995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n correlation ID: b9b321eb-17a4-4f82-91e4-993a13652950\n client request ID: f220b5f7-1c64-40ab-bc96-b0e5ef27428d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xdbc40b14b80\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9ndqj45cmqg2\" to finish creating: GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"name\": \"a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:15:40.641153712Z\",\n \"endTime\": \"2026-10-19T18:44:47.329454995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n correlation ID: b9b321eb-17a4-4f82-91e4-993a13652950\n client request ID: f220b5f7-1c64-40ab-bc96-b0e5ef27428d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-bbzfqrxldsr2/cluster-candidate-5-1-vz4k7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x399878fce5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-vz4k7j: failed waiting for cluster=\"cluster-candidate-5-1-vz4k7j\" in resourcegroup=\"rg-candidate-5-1-bbzfqrxldsr2\" to finish creating: GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"name\": \"f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:52:45.887911577Z\",\n \"endTime\": \"2026-10-19T17:21:46.629500013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n correlation ID: 4cf69c21-e6dd-4c6d-8bd7-3644b7285946\n client request ID: 2f22a1ad-d176-4f31-9493-27c1ef58e861\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-bbzfqrxldsr2/cluster-candidate-5-1-vz4k7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x399878fce5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-vz4k7j: failed waiting for cluster=\"cluster-candidate-5-1-vz4k7j\" in resourcegroup=\"rg-candidate-5-1-bbzfqrxldsr2\" to finish creating: GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"name\": \"f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:52:45.887911577Z\",\n \"endTime\": \"2026-10-19T17:21:46.629500013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n correlation ID: 4cf69c21-e6dd-4c6d-8bd7-3644b7285946\n client request ID: 2f22a1ad-d176-4f31-9493-27c1ef58e861\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"d9779475fed33830e4dd8ea5b21ff77ea27177e1e5ee96139b3ecd5a818f11f6","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"a1239d18a63687ad99406b4ccecdd81e2142d951763a6d66f896f64e917e539e","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","signature_id":"2f9ef72fed1ff555c11644ab75f32dedc822bf7be21ab5ff05bf932afe387496","pr_number":7601,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"5b8322b2740f813049ff2c9cf50d132e24ae2a2035200cb00221f24e6219e8d4","pr_number":7603,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"7ccb09f363018848f8b97a30c2077568c410258b8f6ab2c8e3c4e277049bd960","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.434782608695652,"anchor_occurrences":5,"after_last_push_count":11,"trend_counts":[2,2,2,0,0,0,5],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":46,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:28:50Z\n Ended: 2026-10-19T22:55:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"43f3c967-dd75-47e1-aa51-7f18084ed726\"\n Description: Pod ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T22:29:50Z\n Ended: 2026-10-19T22:55:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"43f3c967-dd75-47e1-aa51-7f18084ed726\"\n Description: Pod ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:36:54Z\n Ended: 2026-10-19T19:06:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ab10566e-97b4-4b52-af7f-f5418105b4d1\"\n Description: Pod ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:36:54Z\n Ended: 2026-10-19T19:06:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ab10566e-97b4-4b52-af7f-f5418105b4d1\"\n Description: Pod ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T17:17:20Z\n Ended: 2026-10-19T17:42:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"001c2a8a-fc34-4e11-96e9-d8a9bf0c0d4a\"\n Description: Pod ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T17:17:20Z\n Ended: 2026-10-19T17:42:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"001c2a8a-fc34-4e11-96e9-d8a9bf0c0d4a\"\n Description: Pod ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"3eb395fbe55fbf76ecbc5b29bc93126f665b5c614d9cc51af8cf407abbc2351a","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"3ec9b5e5b328b5cd8a400942079982d969a6b5748d54871952c045857225f006","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","signature_id":"aac13e9e48b4294d601642f38cf4f1698fe3e6c046c28a9a2ba8ffbab5af5f4a","pr_number":7601,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"f60e38f83fd59b407778f35f45c625f2f225ee28604ad8f0b54ef8309ee41e82","pr_number":7603,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"9f646690f490830019c22ee82a99cd35c179f01219b83c3accb6f681e2db6fe2","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.434782608695652,"anchor_occurrences":5,"after_last_push_count":9,"trend_counts":[3,2,1,0,0,0,5],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":48,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:25:56Z\n Ended: 2026-10-19T22:51:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T22:25:56Z\n Ended: 2026-10-19T22:51:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:34:58Z\n Ended: 2026-10-19T19:02:56Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:35:57Z\n Ended: 2026-10-19T19:06:58Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T17:14:26Z\n Ended: 2026-10-19T17:38:23Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8l75h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T17:26:26Z\n Ended: 2026-10-19T17:38:23Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8l75h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"e78bd233dcf87e792b6e12ee3292aaea83b0ab781757136a8fe46caec4bd397e","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"8e8af42bde7325022e45fef0ceec74e04a08f47685cabbc3285f17cf575d7f3a","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","signature_id":"9c82a77768c5119269831d8f975426ffe0af91bf48ce234e95f2caa15a644081","pr_number":7601,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","signature_id":"44fdc585ddfc610cc29e0362825162ffafcbac3e8a2962c6caf3f0426cf09c09","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"985772ce6b4a5fa9118e8632b816d88f49438c32da40ccd8c2405969b1eff3af","pr_number":7603,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.3478260869565215,"anchor_occurrences":4,"after_last_push_count":28,"trend_counts":[6,11,9,0,0,0,4],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":92,"prior_last_seen_at":"2026-10-15T19:10:04Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:40:56Z\n Ended: 2026-10-19T22:52:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:49:54Z\n Ended: 2026-10-19T19:02:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:52:56Z\n Ended: 2026-10-19T19:03:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T13:54:50Z\n Ended: 2026-10-19T14:07:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T13:55:53Z\n Ended: 2026-10-19T14:07:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"5a6e4b1077d837f8d74d606b27a4e172480813383d0ec67371f063d887422ebe","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"bb91b2c59536dad8e832118575f18644aa9a309c7f2cde3ce9c9ded8d0c6d690","pr_number":7566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"e079f4902db633dcc96bb8366e4fd56a8e30fed4d0a804d743ad8620e1a6ed12","pr_number":7603,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"5823d7990345a57a295b5c723408757e3fd02ee10ebd01906262ea2620d0d364","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.3478260869565215,"anchor_occurrences":4,"after_last_push_count":39,"trend_counts":[7,5,4,2,0,0,4],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":131,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbabb28f39e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-zk8hdsmg8dfw\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"name\": \"7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:39:28.091453996Z\",\n \"endTime\": \"2026-10-20T00:58:30.991397546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n correlation ID: 8265cd70-ed50-4525-a687-45c965024ae6\n client request ID: b95e9e3d-a0f7-4ee8-8ac7-2ff5d62a1d3a\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbabb28f39e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-zk8hdsmg8dfw\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"name\": \"7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:39:28.091453996Z\",\n \"endTime\": \"2026-10-20T00:58:30.991397546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n correlation ID: 8265cd70-ed50-4525-a687-45c965024ae6\n client request ID: b95e9e3d-a0f7-4ee8-8ac7-2ff5d62a1d3a","fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcdba5b46020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"name\": \"f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T21:20:53.14343965Z\",\n \"endTime\": \"2026-10-19T21:40:01.878429551Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n correlation ID: 85a1b8fd-965d-4893-90c4-5bc86f87328d\n client request ID: 6c4ed168-8339-4ad9-9c7e-dcd04bbc9a76\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcdba5b46020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"name\": \"f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T21:20:53.14343965Z\",\n \"endTime\": \"2026-10-19T21:40:01.878429551Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n correlation ID: 85a1b8fd-965d-4893-90c4-5bc86f87328d\n client request ID: 6c4ed168-8339-4ad9-9c7e-dcd04bbc9a76","fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39c1da55a940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-vfc45fcxtxqz\" to finish creating: GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"name\": \"98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:03:22.205099398Z\",\n \"endTime\": \"2026-10-19T18:22:25.549095315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n correlation ID: 697bbb7b-4488-43cf-b80e-bef1ce56a7c0\n client request ID: 9c6f432c-bb1f-45d1-a1cc-acc4672dbd73\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39c1da55a940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-vfc45fcxtxqz\" to finish creating: GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"name\": \"98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:03:22.205099398Z\",\n \"endTime\": \"2026-10-19T18:22:25.549095315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n correlation ID: 697bbb7b-4488-43cf-b80e-bef1ce56a7c0\n client request ID: 9c6f432c-bb1f-45d1-a1cc-acc4672dbd73"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","signature_id":"1c4f6dbd6e184b39b789fddbe55d13e4716fae9f427dad99450ca3f073344d50","pr_number":7621,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112275231841193984","occurred_at":"2026-10-19T20:10:34Z","signature_id":"a5f3943cfdc00ecde813bc0fb08ef3bd16b3ef30062a43d942be7abcdafd19ff","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112229674405859328","occurred_at":"2026-10-19T17:09:32Z","signature_id":"820aba5d0f7b24f476521876d79e79e741d888ec783781b82502d68ebbdb843e","pr_number":6191,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112019392656052224","occurred_at":"2026-10-19T03:13:57Z","signature_id":"eb8dcd63f6fa8106b7aefdf7658de4b7ab552e999feda4f0ea87bdb655d6430c","pr_number":7504,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.260869565217391,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-16T07:17:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-sjxzmq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0c69717cc0\u003e: \n DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e0c6a7f4740\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-sjxzmq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0c69717cc0\u003e: \n DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e0c6a7f4740\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a6b532d3880\u003e: \n DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a6b532d22c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a6b532d3880\u003e: \n DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a6b532d22c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-2qp7lk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd92a3e87160\u003e: \n DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd92a3f627c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-2qp7lk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd92a3e87160\u003e: \n DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd92a3f627c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","signature_id":"88045e20d4aa89c6b9271f51a87c8e3f4bbca7bf37da9e0736de49e88fc8b858","pr_number":7301,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112168271166836736","occurred_at":"2026-10-19T13:05:30Z","signature_id":"db8a48ab40d96ea4fb53c96ba6b139542335826edd428ef0ac83fd3e9bc63e79","pr_number":7604,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112139306637201408","occurred_at":"2026-10-19T11:10:26Z","signature_id":"5d44285e476fc25f2fca3fbecb4c19176951c7cc91f18773f8fc9aab98a817d1","pr_number":7601,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.260869565217391,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[3,2,7,2,0,0,3],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-16T09:12:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0051281e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6h69n8f46csz\" to finish creating: GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\",\n \"name\": \"65c67213-a430-416b-a8e1-9c576702841f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:03:54.135232982Z\",\n \"endTime\": \"2026-10-19T23:22:56.513764235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n correlation ID: d01ca859-c416-4ccc-bd23-51544b0af025\n client request ID: 937df210-9f80-46da-9e07-0166c55c864d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0051281e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6h69n8f46csz\" to finish creating: GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\",\n \"name\": \"65c67213-a430-416b-a8e1-9c576702841f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:03:54.135232982Z\",\n \"endTime\": \"2026-10-19T23:22:56.513764235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n correlation ID: d01ca859-c416-4ccc-bd23-51544b0af025\n client request ID: 937df210-9f80-46da-9e07-0166c55c864d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d397c5fe960\u003e: \n failed to create NodePool np-24dvjs: failed waiting for nodepool=\"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\" in resourcegroup=\"rg-hs-pre--qmdx974n6w26\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"name\": \"8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:14:24.493403189Z\",\n \"endTime\": \"2026-10-19T22:33:34.182492247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n correlation ID: 33ce383b-ea0f-4d30-8902-6098f7bbfeb8\n client request ID: 9ce1f451-6c72-4c5f-8183-8c4490de781d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d397c5fe960\u003e: \n failed to create NodePool np-24dvjs: failed waiting for nodepool=\"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\" in resourcegroup=\"rg-hs-pre--qmdx974n6w26\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"name\": \"8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:14:24.493403189Z\",\n \"endTime\": \"2026-10-19T22:33:34.182492247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n correlation ID: 33ce383b-ea0f-4d30-8902-6098f7bbfeb8\n client request ID: 9ce1f451-6c72-4c5f-8183-8c4490de781d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b24ac0020c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p69pvshqvq9p\" to finish creating: GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"name\": \"c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T15:56:13.653725725Z\",\n \"endTime\": \"2026-10-19T16:15:18.87162238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n correlation ID: 5cc9d41d-adc2-4131-802d-17d3ab484012\n client request ID: b9e380f6-f1bc-4da7-bbfb-dbb4f50cb471\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b24ac0020c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p69pvshqvq9p\" to finish creating: GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"name\": \"c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T15:56:13.653725725Z\",\n \"endTime\": \"2026-10-19T16:15:18.87162238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n correlation ID: 5cc9d41d-adc2-4131-802d-17d3ab484012\n client request ID: b9e380f6-f1bc-4da7-bbfb-dbb4f50cb471\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302045275885568","occurred_at":"2026-10-19T21:57:06Z","signature_id":"c1c99568d26a1ef70092e8ca69e04aa48a183d5691882b98ba81908b32e59ff0","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"b7c7573f03c8b34839471a9a294fd246154a93ada5e8f67ea9a5a4f9e6427d6b","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112190987718627328","occurred_at":"2026-10-19T14:35:30Z","signature_id":"857ad3c84c83c23ff39c0a1984f03a5c20f67f7eae51b6788cb5159fcdd164ca","pr_number":7250,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.260869565217391,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,1,0,0,3],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":24,"prior_last_seen_at":"2026-10-16T13:42:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc80b6d518e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc80b6d518e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xaeb08df4660\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xaeb08df4660\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.009s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x207ea69dfda0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.009s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x207ea69dfda0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","signature_id":"e1dc04bb7befa9011c60269ec95a0b379daa505a076d651f252178142e2c1a97","pr_number":7621,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321203636015104","occurred_at":"2026-10-19T23:12:44Z","signature_id":"59dc8637178db724ee538e803f30db4a90c31b43e902905b89afa3e5cf02ff0b","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","signature_id":"af83b0a23ad7b87be6df260bf29be76315509cb2eae7c8baf1ce1bbbbc9b604b","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-06T05:43:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:437]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x340bb99d4c0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rhzhzsd2dvs2\" to finish creating: GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"name\": \"1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:28:35.166007052Z\",\n \"endTime\": \"2026-10-19T20:47:38.665697527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n correlation ID: da09f0ee-1b1c-4dd0-986e-72ed6522e248\n client request ID: 773c74d7-3374-4a99-9619-c2776f1df3e0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:437]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x340bb99d4c0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rhzhzsd2dvs2\" to finish creating: GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"name\": \"1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:28:35.166007052Z\",\n \"endTime\": \"2026-10-19T20:47:38.665697527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n correlation ID: da09f0ee-1b1c-4dd0-986e-72ed6522e248\n client request ID: 773c74d7-3374-4a99-9619-c2776f1df3e0\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1051bd7af500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-p7tdc2\" in resourcegroup=\"rg-np-version-upgrade-p7tdc2-lnnmsq2r4w9h\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"name\": \"70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:02:00.685391332Z\",\n \"endTime\": \"2026-10-19T17:21:08.269829146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n correlation ID: fb9e2f48-428d-4f30-8c61-d916aa767023\n client request ID: 93ac57ce-9b86-4ca4-8a14-339c496ee50f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1051bd7af500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-p7tdc2\" in resourcegroup=\"rg-np-version-upgrade-p7tdc2-lnnmsq2r4w9h\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"name\": \"70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:02:00.685391332Z\",\n \"endTime\": \"2026-10-19T17:21:08.269829146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n correlation ID: fb9e2f48-428d-4f30-8c61-d916aa767023\n client request ID: 93ac57ce-9b86-4ca4-8a14-339c496ee50f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112266775457435648","occurred_at":"2026-10-19T19:36:57Z","signature_id":"bdbb32bcebb5a63a8a31b16a6425f1816af2a8db1eb1c49c5c7f8669296625b2","pr_number":7614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","signature_id":"aebdc6168daad45cbe0abf25f4b28797a97898ae2a5fbc5180889ca8a292b42e","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":12,"trend_counts":[1,4,5,1,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":22,"prior_last_seen_at":"2026-10-16T13:20:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17293bf48060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-qw8hmnl947sz\" to finish creating: GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"name\": \"2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:54:37.149407446Z\",\n \"endTime\": \"2026-10-20T00:13:41.899452027Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n correlation ID: a4d6efb0-533f-4043-ad67-651fe433c379\n client request ID: 6f8a1c34-923e-40e5-a5b3-733c67d8d88f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17293bf48060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-qw8hmnl947sz\" to finish creating: GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"name\": \"2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:54:37.149407446Z\",\n \"endTime\": \"2026-10-20T00:13:41.899452027Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n correlation ID: a4d6efb0-533f-4043-ad67-651fe433c379\n client request ID: 6f8a1c34-923e-40e5-a5b3-733c67d8d88f\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83e42499e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-85n46zxr29bc\" to finish creating: GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\",\n \"name\": \"eb476919-259d-4581-ba0c-41803ac281da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:15:51.295694688Z\",\n \"endTime\": \"2026-10-19T17:34:52.47411646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n correlation ID: 2736a12e-0c94-445a-972d-f60f128898b0\n client request ID: 8ae6dbe5-3e36-4ee4-b803-b4d4643b03d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83e42499e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-85n46zxr29bc\" to finish creating: GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\",\n \"name\": \"eb476919-259d-4581-ba0c-41803ac281da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:15:51.295694688Z\",\n \"endTime\": \"2026-10-19T17:34:52.47411646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n correlation ID: 2736a12e-0c94-445a-972d-f60f128898b0\n client request ID: 8ae6dbe5-3e36-4ee4-b803-b4d4643b03d4\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","signature_id":"6f7bc0014c51cfc8989a4b4711871dc6b12df8b82d9220ab61461f581c9560eb","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112212933982294016","occurred_at":"2026-10-19T16:02:58Z","signature_id":"eb441e563244116bfa29f1b8638f19b4aa62c4d3fab18f60eecf424b95ccceca","pr_number":7613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-14T18:34:47Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-10-19T18:44:25.577Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-19T18:44:27.375Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-19T18:44:29.162Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8937216-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f\ntime=2026-10-19T18:53:01.241Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Resources/deployments/6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f/operationStatuses/08584091714172679540\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Resources/deployments/profile-w5ahaumc2w6ei\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j8937216-svc/resourceAssociations/nsp-j8937216-svc-ra-d5a635c9-43bf-5de7-aaf2-5232b9ebe5d0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j8937216-svc/resourceAssociations/nsp-j8937216-svc-ra-d5a635c9-43bf-5de7-aaf2-5232b9ebe5d0, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-19T14:43:12.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 description=\"Step nsp\\n Kind: ARM\\n Template: templates/mgmt-nsp.bicep\\n Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-19T14:43:13.235Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1\ntime=2026-10-19T14:43:13.684Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 deployment=299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1534464-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0\ntime=2026-10-19T14:43:43.960Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Resources/deployments/299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0/operationStatuses/08584091858920261215\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Resources/deployments/profile-xcn2acurowiq2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j1534464-mgmt-1/resourceAssociations/nsp-j1534464-mgmt-1-ra-66bb4858-d2d4-5bd8-8103-db26bfc7bd94\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j1534464-mgmt-1/resourceAssociations/nsp-j1534464-mgmt-1-ra-66bb4858-d2d4-5bd8-8103-db26bfc7bd94, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112247946668937216","occurred_at":"2026-10-19T18:22:08Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7602,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112181488601534464","occurred_at":"2026-10-19T13:58:03Z","signature_id":"6c11c44592820325fa1934724e67fb473057c5c4359853d87ec5141e3481ef08","pr_number":7592,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,0,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-14T18:32:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x13c01da76240\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x13c01e312060\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01f5045e0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01e3822a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x13c01da76240\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x13c01e312060\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01f5045e0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01e3822a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1ffd80fcc600\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1ffd80c0ac80\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd811f6800\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd80d0e5e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1ffd80fcc600\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1ffd80c0ac80\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd811f6800\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd80d0e5e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112266775457435648","occurred_at":"2026-10-19T19:36:57Z","signature_id":"eeda0a68876291ebbf928a7c660b039924edb75924c1c795e3d5cba0f09af203","pr_number":7614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","signature_id":"fed984ab46e663f21fa5a7bc5ab953f16af1cfb5d9265996c796fa791e841180","pr_number":7301,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,1,1,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-15T01:36:32Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-19T19:36:46.694Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-19T19:36:47.222Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-19T19:36:47.222Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:36:47.440Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-19T19:36:47.554Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-19T19:36:47.554Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:36:47.600Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:42:29.171Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T19:42:29.171Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-19T15:56:31.916Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-19T15:56:32.339Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-19T15:56:32.339Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T15:56:32.526Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-19T15:56:32.659Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-19T15:56:32.659Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T15:56:32.731Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T16:02:20.224Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T16:02:20.224Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112259193590255616","occurred_at":"2026-10-19T19:06:50Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112203264580653056","occurred_at":"2026-10-19T15:24:35Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e15f9141b2e42a01158494cd605cc8736f505f4288f28a00af49203db3d9fd9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x22128cbeb10\u003e: \n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"name\": \"26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:37.259328289Z\",\n \"endTime\": \"2026-10-19T11:01:44.306693815Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-node-tuning-operator, certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, dns-operator, ingress-operator, redhat-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n correlation ID: 87b43167-8612-41cf-b2af-01e82520620b\n client request ID: 826cfb23-3364-490c-b9f1-549ca623afb4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x22128cbeb10\u003e: \n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"name\": \"26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:37.259328289Z\",\n \"endTime\": \"2026-10-19T11:01:44.306693815Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-node-tuning-operator, certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, dns-operator, ingress-operator, redhat-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n correlation ID: 87b43167-8612-41cf-b2af-01e82520620b\n client request ID: 826cfb23-3364-490c-b9f1-549ca623afb4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22a4ab41e440\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-tkvqmscnx8hd\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"name\": \"7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:11.987681126Z\",\n \"endTime\": \"2026-10-19T11:01:14.305309512Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n correlation ID: 8f412970-2932-49aa-85c1-e501fdb88128\n client request ID: 037bff49-0293-4f3d-ab7f-9ff8b71e3d4a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22a4ab41e440\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-tkvqmscnx8hd\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"name\": \"7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:11.987681126Z\",\n \"endTime\": \"2026-10-19T11:01:14.305309512Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n correlation ID: 8f412970-2932-49aa-85c1-e501fdb88128\n client request ID: 037bff49-0293-4f3d-ab7f-9ff8b71e3d4a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"8e175b76a31d7caa36a30a13af8ff7ce54e783d826c89b8e5d1babc7577f83a0","pr_number":7582,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"cb2937c580cb5b7a220440cf73792f0de178b090817c5fbd61ec46b3faf7c8f1","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,2],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T16:43:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-bkqnpw2l8zf5/cluster-candidate-5-0-ccpvpc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d23d4310dc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ccpvpc: failed waiting for cluster=\"cluster-candidate-5-0-ccpvpc\" in resourcegroup=\"rg-candidate-5-0-bkqnpw2l8zf5\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"name\": \"44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:41.722981857Z\",\n \"endTime\": \"2026-10-19T11:00:44.465792588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n correlation ID: 77467a09-84cb-420c-a3bc-027f97f67194\n client request ID: 2b188110-77bd-4bc4-9d94-27a812b7e09a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-bkqnpw2l8zf5/cluster-candidate-5-0-ccpvpc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d23d4310dc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ccpvpc: failed waiting for cluster=\"cluster-candidate-5-0-ccpvpc\" in resourcegroup=\"rg-candidate-5-0-bkqnpw2l8zf5\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"name\": \"44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:41.722981857Z\",\n \"endTime\": \"2026-10-19T11:00:44.465792588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n correlation ID: 77467a09-84cb-420c-a3bc-027f97f67194\n client request ID: 2b188110-77bd-4bc4-9d94-27a812b7e09a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xd36763f9420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q6wh6zbcv2x2\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\",\n \"name\": \"42423002-b716-49bf-b44c-2e1fd4733997\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.508047813Z\",\n \"endTime\": \"2026-10-19T11:00:34.28001697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n correlation ID: 1aed53cb-56a1-406f-8bbf-9de9f16e4e1c\n client request ID: 90f415c0-66a9-4a24-b9de-2db2faf18c9c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xd36763f9420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q6wh6zbcv2x2\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\",\n \"name\": \"42423002-b716-49bf-b44c-2e1fd4733997\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.508047813Z\",\n \"endTime\": \"2026-10-19T11:00:34.28001697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n correlation ID: 1aed53cb-56a1-406f-8bbf-9de9f16e4e1c\n client request ID: 90f415c0-66a9-4a24-b9de-2db2faf18c9c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"b208af379d964ed16647d5f8ea8d0ca59bb7ee9c6c4869a7d77f00e95f411925","pr_number":7582,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"e4c0b02a4cead83c39b9290d8769844d1c05d2d84446e896f711e6903b9a464b","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4040a72f14f446e40df065387b843c2ab59e8482c18302866d9ef52177992739","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after ad... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2ba9acdcc318\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-pp8lljnf5t7j\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"name\": \"025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:45:45.996504863Z\",\n \"endTime\": \"2026-10-19T17:04:48.768522414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-7v7tf-6wgpp: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-pp8lljnf5t7j--managed/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-pp8lljnf5t7j--managed/providers/Microsoft.Compute/availabilitySets/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after ad... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n correlation ID: c38c42ba-d929-4393-9a38-4d460e6cc1c1\n client request ID: 991ecdc6-7a24-4e68-a023-12a15de7fc26\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2ba9acdcc318\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-pp8lljnf5t7j\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"name\": \"025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:45:45.996504863Z\",\n \"endTime\": \"2026-10-19T17:04:48.768522414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-7v7tf-6wgpp: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-pp8lljnf5t7j--managed/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-pp8lljnf5t7j--managed/providers/Microsoft.Compute/availabilitySets/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after ad... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n correlation ID: c38c42ba-d929-4393-9a38-4d460e6cc1c1\n client request ID: 991ecdc6-7a24-4e68-a023-12a15de7fc26\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","signature_id":"0217128493b7828f90a93261926d540dcdbdc6f3405b87a78fddf4c7104ab055","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"618da575c5b58fd2ef178dda38409b6151172a0ed224c27153d5f47fde89e12b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional token... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xacc0f2d2440\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jlnfb8x8vq5b\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"name\": \"d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:29:55.150212587Z\",\n \"endTime\": \"2026-10-19T13:48:55.883771022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-fgcc4 to exist\\nMachine ephemeral-disk-ephemeral-np-pscp5-w4psz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-w4psz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-jlnfb8x8vq5b--managed/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n correlation ID: 346dd7ce-e606-492f-925a-aa985115ed46\n client request ID: 9eb4247b-08f9-4cb3-a183-7df951ef2271\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xacc0f2d2440\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jlnfb8x8vq5b\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"name\": \"d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:29:55.150212587Z\",\n \"endTime\": \"2026-10-19T13:48:55.883771022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-fgcc4 to exist\\nMachine ephemeral-disk-ephemeral-np-pscp5-w4psz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-w4psz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-jlnfb8x8vq5b--managed/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n correlation ID: 346dd7ce-e606-492f-925a-aa985115ed46\n client request ID: 9eb4247b-08f9-4cb3-a183-7df951ef2271\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","signature_id":"77845656a2d51a0f7208a5af61597929bc726757f687ffa28f72cf3f1c408b72","pr_number":7603,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0ba247174d89d7a38b8deaaec8673bd9296102e3e139d937d6b4d7517e364180","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d7273ae000\u003e: \n failed to create NodePool np-vkgl4p: failed waiting for nodepool=\"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\" in resourcegroup=\"rg-hs-pre--j799lrwqxz6j\" to finish creating: GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"name\": \"c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:21:51.863828648Z\",\n \"endTime\": \"2026-10-19T22:40:54.499105259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: Failed: virtualmachine failed to create or update. err: failed to get existing resource rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n correlation ID: 2e542bba-f131-4b5d-bf4c-a77a8bc95e2d\n client request ID: 83a4b938-e107-403c-83e6-0592ebdaeff2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d7273ae000\u003e: \n failed to create NodePool np-vkgl4p: failed waiting for nodepool=\"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\" in resourcegroup=\"rg-hs-pre--j799lrwqxz6j\" to finish creating: GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"name\": \"c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:21:51.863828648Z\",\n \"endTime\": \"2026-10-19T22:40:54.499105259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: Failed: virtualmachine failed to create or update. err: failed to get existing resource rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n correlation ID: 2e542bba-f131-4b5d-bf4c-a77a8bc95e2d\n client request ID: 83a4b938-e107-403c-83e6-0592ebdaeff2\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112290330521899008","occurred_at":"2026-10-19T21:10:33Z","signature_id":"788f52be295e2db0098488b52e436a5a34110ee9698b2cbb0ae2a28aed2ec616","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fe322c7e500\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2qqsdjg4hsrq\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"name\": \"7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:36:56.91702797Z\",\n \"endTime\": \"2026-10-20T00:55:57.103083611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh to exist\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-gl748: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-gl748 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-2qqsdjg4hsrq--managed/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Network/networkInterfaces/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to htt... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n correlation ID: 2083ceae-e353-4ba6-a1e3-ff92ee472926\n client request ID: 9c9e7813-8848-4608-b832-b1dbe65ca4ad\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fe322c7e500\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2qqsdjg4hsrq\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"name\": \"7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:36:56.91702797Z\",\n \"endTime\": \"2026-10-20T00:55:57.103083611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh to exist\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-gl748: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-gl748 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-2qqsdjg4hsrq--managed/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Network/networkInterfaces/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to htt... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n correlation ID: 2083ceae-e353-4ba6-a1e3-ff92ee472926\n client request ID: 9c9e7813-8848-4608-b832-b1dbe65ca4ad\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","signature_id":"97312b6eb167e1169037b6c71d59e152ec18721dd3caa446c5b8a711966bf072","pr_number":7621,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6922af18a79657334aba15a6dc66a7eed9b7eb1fb78e4dc25e99132916eaecff","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x169a1fa14360\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pvwnm5h2lwhh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"name\": \"201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:39.240419381Z\",\n \"endTime\": \"2026-10-19T11:00:44.480106321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, catalog-operator, cluster-network-operator, certified-operators-catalog, olm-collect-profiles, openshift-controller-manager, packageserver, machine-approver, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, openshift-route-controller-manager, community-operators-catalog, cluster-image-registry-operator, dns-operator, konnectivity-agent, olm-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n correlation ID: a21d9aa7-da4e-4f79-9da1-353faa5bb8d0\n client request ID: 457b6d19-40ff-4036-a44f-729e35535125\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x169a1fa14360\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pvwnm5h2lwhh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"name\": \"201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:39.240419381Z\",\n \"endTime\": \"2026-10-19T11:00:44.480106321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, catalog-operator, cluster-network-operator, certified-operators-catalog, olm-collect-profiles, openshift-controller-manager, packageserver, machine-approver, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, openshift-route-controller-manager, community-operators-catalog, cluster-image-registry-operator, dns-operator, konnectivity-agent, olm-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n correlation ID: a21d9aa7-da4e-4f79-9da1-353faa5bb8d0\n client request ID: 457b6d19-40ff-4036-a44f-729e35535125\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"14ccb9df6630ab0c7d18ff56028197d76162b4a16d38963003971697eb3da1eb","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-28T04:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-ld4f2qxz9tlh/cluster-candidate-4-23-prg9qx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x103e49a6a6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-prg9qx: failed waiting for cluster=\"cluster-candidate-4-23-prg9qx\" in resourcegroup=\"rg-candidate-4-23-ld4f2qxz9tlh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"name\": \"87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.32598821Z\",\n \"endTime\": \"2026-10-19T11:00:34.343098485Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-autoscaler, cluster-image-registry-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, catalog-operator, kube-storage-version-migrator, hosted-cluster-config-operator, olm-collect-profiles, konnectivity-agent, kube-scheduler, machine-approver, cluster-storage-operator, certified-operators-catalog, cluster-node-tuning-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, kube-controller-manager, olm-operator, control-plane-pki-operator, ingress-operator, cluster-version-operator, openshift-apiserver, cluster-policy-controller, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n correlation ID: fc208c85-0c22-4fd2-b0d8-868115a21156\n client request ID: 2c46da4c-f387-4bd8-8b83-56d49c4e1b76\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-ld4f2qxz9tlh/cluster-candidate-4-23-prg9qx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x103e49a6a6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-prg9qx: failed waiting for cluster=\"cluster-candidate-4-23-prg9qx\" in resourcegroup=\"rg-candidate-4-23-ld4f2qxz9tlh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"name\": \"87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.32598821Z\",\n \"endTime\": \"2026-10-19T11:00:34.343098485Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-autoscaler, cluster-image-registry-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, catalog-operator, kube-storage-version-migrator, hosted-cluster-config-operator, olm-collect-profiles, konnectivity-agent, kube-scheduler, machine-approver, cluster-storage-operator, certified-operators-catalog, cluster-node-tuning-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, kube-controller-manager, olm-operator, control-plane-pki-operator, ingress-operator, cluster-version-operator, openshift-apiserver, cluster-policy-controller, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n correlation ID: fc208c85-0c22-4fd2-b0d8-868115a21156\n client request ID: 2c46da4c-f387-4bd8-8b83-56d49c4e1b76\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"9d277ac26f53ad4a08cdfd07a5091cd11c8dfc2c77ed5fa68ca2993e5b928331","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-29T01:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe67dc5ca00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gs2cw8rqvtsp\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"name\": \"9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:35.601658411Z\",\n \"endTime\": \"2026-10-19T11:02:44.344881038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, ingress-operator, olm-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, kube-controller-manager, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-policy-controller, azure-cloud-controller-manager, packageserver, cluster-version-operator, openshift-apiserver, cluster-autoscaler, machine-approver, capi-provider, catalog-operator, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n correlation ID: ebcf2a7d-e9a4-4dbb-b982-a31f966e3404\n client request ID: d11643b0-b580-4f4f-a16e-c60b01f474c0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe67dc5ca00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gs2cw8rqvtsp\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"name\": \"9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:35.601658411Z\",\n \"endTime\": \"2026-10-19T11:02:44.344881038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, ingress-operator, olm-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, kube-controller-manager, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-policy-controller, azure-cloud-controller-manager, packageserver, cluster-version-operator, openshift-apiserver, cluster-autoscaler, machine-approver, capi-provider, catalog-operator, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n correlation ID: ebcf2a7d-e9a4-4dbb-b982-a31f966e3404\n client request ID: d11643b0-b580-4f4f-a16e-c60b01f474c0\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"9cf94771ae7bdae1859fe77a8ea6f9b3c3f1d43f1721da6e61995c2a20f5059c","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"854689b6465ea9b1f99065d40b48241478ad62957ce512515a9e3318a109b484","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4jxfrx\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b2f64940d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4jxfrx: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4jxfrx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dxmr9r2kr8sj\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"name\": \"e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:38.502730447Z\",\n \"endTime\": \"2026-10-19T11:00:44.46660972Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n correlation ID: 72780af9-37e8-42d0-b405-3ed5aea87756\n client request ID: 04371127-7fbb-474b-a1aa-c21583ae4e63\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4jxfrx\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b2f64940d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4jxfrx: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4jxfrx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dxmr9r2kr8sj\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"name\": \"e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:38.502730447Z\",\n \"endTime\": \"2026-10-19T11:00:44.46660972Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n correlation ID: 72780af9-37e8-42d0-b405-3ed5aea87756\n client request ID: 04371127-7fbb-474b-a1aa-c21583ae4e63\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"fc3a92da63d1f72fca2a38dd8f0204f04d58ef8149a8800bb69f9a022034a922","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"362976a3de1c8de9ef8affd7b179e4e636ba9b64bddffacea53a745a204432e5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28db69b98ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-f6gwn5qzckpq\" to finish creating: GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"name\": \"55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:48:10.999588854Z\",\n \"endTime\": \"2026-10-19T20:07:15.605638501Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-wmvjh-tdsrw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-wmvjh-tdsrw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n correlation ID: 43d991c8-f99d-4e5d-8930-6d417ef255cd\n client request ID: 24277a58-4912-4429-b89c-c6baeb4435fc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28db69b98ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-f6gwn5qzckpq\" to finish creating: GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"name\": \"55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:48:10.999588854Z\",\n \"endTime\": \"2026-10-19T20:07:15.605638501Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-wmvjh-tdsrw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-wmvjh-tdsrw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n correlation ID: 43d991c8-f99d-4e5d-8930-6d417ef255cd\n client request ID: 24277a58-4912-4429-b89c-c6baeb4435fc\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112258228648677376","occurred_at":"2026-10-19T19:03:00Z","signature_id":"53bc6aee73b2dd32359da6c26960a91c8d89c84b41fd03eac07099db143926b5","pr_number":7602,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-16T20:03:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x106d601dad60\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-dgl2pkrhlxx4\" to finish creating: GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"name\": \"87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T14:24:33.557951156Z\",\n \"endTime\": \"2026-10-19T14:43:40.561544029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n correlation ID: 504c59a4-77a5-4f21-a6c2-acde02224bf6\n client request ID: fb1b6d77-0e4c-4919-b3ea-36fd2e07f63c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x106d601dad60\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-dgl2pkrhlxx4\" to finish creating: GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"name\": \"87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T14:24:33.557951156Z\",\n \"endTime\": \"2026-10-19T14:43:40.561544029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n correlation ID: 504c59a4-77a5-4f21-a6c2-acde02224bf6\n client request ID: fb1b6d77-0e4c-4919-b3ea-36fd2e07f63c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","signature_id":"279ce8581e5dd18f302a233fe92088be596d83316a923d7afe6a4828279b16b7","pr_number":7598,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ef1a859d4a5b48ed2f975faa7a87f38129b89e34f695186a2b0e35ba0b81d5df","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3895fbd87c80\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-89sq9p7645tk\" to finish creating: GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"name\": \"111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:38:52.630267131Z\",\n \"endTime\": \"2026-10-19T19:57:56.018733569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n correlation ID: 439e72b2-9c87-4ffb-ac85-0abf45d7f0d2\n client request ID: 07db40a5-8bb3-4389-ac48-a0125b515b0e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3895fbd87c80\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-89sq9p7645tk\" to finish creating: GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"name\": \"111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:38:52.630267131Z\",\n \"endTime\": \"2026-10-19T19:57:56.018733569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n correlation ID: 439e72b2-9c87-4ffb-ac85-0abf45d7f0d2\n client request ID: 07db40a5-8bb3-4389-ac48-a0125b515b0e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112246631993708544","occurred_at":"2026-10-19T18:16:55Z","signature_id":"aaacb1de33b5fad81d04fae31a9da3f8a348ac3d9a71a0bc7e77be76da64316b","pr_number":7616,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-16T09:20:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d6e7c8ac020\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t27g7scwdvdr\" to finish creating: GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"name\": \"04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:30:35.481364162Z\",\n \"endTime\": \"2026-10-19T16:49:44.492413303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n correlation ID: 164cf44e-1a4c-43ef-ad03-502d13c22da5\n client request ID: 949d080f-05a2-40b5-b875-b9fbced5c91f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d6e7c8ac020\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t27g7scwdvdr\" to finish creating: GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"name\": \"04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:30:35.481364162Z\",\n \"endTime\": \"2026-10-19T16:49:44.492413303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n correlation ID: 164cf44e-1a4c-43ef-ad03-502d13c22da5\n client request ID: 949d080f-05a2-40b5-b875-b9fbced5c91f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199925969195008","occurred_at":"2026-10-19T15:11:19Z","signature_id":"d79807bc63d1c36ab2b3708c01de13b2aeba0757391700b53e65ce99e6bbb469","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb1eb44ae687b456506e286274b48afd57d492aeeab49cb4b52c4310e1e06f33","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeDoesNotExist: Node does not exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Deleting: virtualmachine deleting; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f88271a0ea0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2lvnhrzd5g7t\" to finish creating: GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"name\": \"9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:38:57.011707926Z\",\n \"endTime\": \"2026-10-19T16:58:00.847122787Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: Deleting: virtualmachine deleting\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n correlation ID: 78955e10-7740-4b87-a377-073a6ecddbd0\n client request ID: a8ccc39c-63f9-475d-8c01-623fcd9a0f40\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f88271a0ea0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2lvnhrzd5g7t\" to finish creating: GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"name\": \"9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:38:57.011707926Z\",\n \"endTime\": \"2026-10-19T16:58:00.847122787Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: Deleting: virtualmachine deleting\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n correlation ID: 78955e10-7740-4b87-a377-073a6ecddbd0\n client request ID: a8ccc39c-63f9-475d-8c01-623fcd9a0f40\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","signature_id":"f74d9e159d8da00a8b5ec1b1bb62fcea11dcb160c4ea6098a0c46c6cca9dfd86","pr_number":7132,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-18T21:42:26Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-10-19T23:00:45.350Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-19T23:00:46.914Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-19T23:00:47.803Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1281792%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5\ntime=2026-10-19T23:08:19.085Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1281792/providers/Microsoft.Resources/deployments/f66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5/operationStatuses/08584091560383429326\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1281792/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1281792/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j1281792\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 84ce1c4b-104f-4d59-ba97-ca7f687e5f6a, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 50b97f78-d517-42ba-b6b3-2e3807c11c56, Request URI: /apps/56b56b41-1aa1-47de-9428-4a9b9ccbfcc7/services/5492004a-a82d-4653-81b2-013acecf0ba5/partitions/4d300830-d333-45bc-8ad6-8d4a8244255d/replicas/134308569966594579s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-19T23:06:28.2874662Z, RequestEndTime: 2026-10-19T23:06:33.6942116Z, Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:35.8111686Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.582,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21738860.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0143,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:45.8134136Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.599,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21747896.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0589,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":27},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:55.8162151Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.543,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21739028.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0371,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:05.8318538Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.956,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21715984.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.3135,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:15.8338344Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.371,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21721028.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0646,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:25.8342074Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.996,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21715496.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0674,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-19T23:06:28.2874662Z; ResponseTime: 2026-10-19T23:06:33.6942116Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be312.documents.azure.com:11303/apps/56b56b41-1aa1-47de-9428-4a9b9ccbfcc7/services/5492004a-a82d-4653-81b2-013acecf0ba5/partitions/4d300830-d333-45bc-8ad6-8d4a8244255d/replicas/134308569966594579s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 5410.799, ActivityId: 50b97f78-d517-42ba-b6b3-2e3807c11c56, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11303 | status: Connected | lkt: 10/19/2026 11:06:33 PM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874662Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0013},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874675Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0046},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874721Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0625},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2875346Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 5411.5582},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:33.6990928Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0545},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:33.6991473Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:15.0012932Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:15.0012932Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:16.0332114Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112311076581281792","occurred_at":"2026-10-19T22:33:00Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7621,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[2,1,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-14T19:26:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: failed to create customer resources for second cluster \"basic-hcp-cluster2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cfd776cc680\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-vrgp2b\" in resourcegroup=\"customer-rg-d7bzdbm7gf6t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xooij5h335npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n correlation ID: 0edba454-3af2-4d43-98da-9b281d7b5b9e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: failed to create customer resources for second cluster \"basic-hcp-cluster2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cfd776cc680\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-vrgp2b\" in resourcegroup=\"customer-rg-d7bzdbm7gf6t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xooij5h335npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n correlation ID: 0edba454-3af2-4d43-98da-9b281d7b5b9e\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","signature_id":"377d1e8a5ea363b1d3314933ad693d46789868ec6ddcce7416d4a3a08af5b8ff","pr_number":7566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":16,"trend_counts":[12,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":134,"prior_last_seen_at":"2026-10-13T15:48:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3d4749b3c078\u003e: \n resource group \"customer-rg-j9j4x64vc846\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-j9j4x64vc846\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"name\": \"060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:02:54.979353355Z\",\n \"endTime\": \"2026-10-19T20:27:00.270376787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ti2ch5ekujd757n62ntgfdpp1p6fpca still exists (deletion dispatched at 2026-10-19T20:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n correlation ID: 1b181c6c-5659-46ac-a08f-39fecbba1c31\n client request ID: 190e259b-25d3-414f-bd78-f91c53d9c681\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3d4749b3c078\u003e: \n resource group \"customer-rg-j9j4x64vc846\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-j9j4x64vc846\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"name\": \"060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:02:54.979353355Z\",\n \"endTime\": \"2026-10-19T20:27:00.270376787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ti2ch5ekujd757n62ntgfdpp1p6fpca still exists (deletion dispatched at 2026-10-19T20:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n correlation ID: 1b181c6c-5659-46ac-a08f-39fecbba1c31\n client request ID: 190e259b-25d3-414f-bd78-f91c53d9c681\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260148863963136","occurred_at":"2026-10-19T19:10:37Z","signature_id":"b4ca3862e521c03d799aa9ddf1252fcdf206a45876ae63817e25772e37f22022","pr_number":7497,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x424386240f0\u003e: \n resource group \"delayed-rbac-9xs9kgbdmnwv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-9xs9kgbdmnwv\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"name\": \"2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:53:00.471686176Z\",\n \"endTime\": \"2026-10-19T20:17:09.335861904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n correlation ID: e5bf541d-883b-4a45-98e4-199dc9b858a9\n client request ID: 39529ecb-f70a-4c21-a2de-abd0e5ac1e6b\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x424386240f0\u003e: \n resource group \"delayed-rbac-9xs9kgbdmnwv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-9xs9kgbdmnwv\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"name\": \"2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:53:00.471686176Z\",\n \"endTime\": \"2026-10-19T20:17:09.335861904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n correlation ID: e5bf541d-883b-4a45-98e4-199dc9b858a9\n client request ID: 39529ecb-f70a-4c21-a2de-abd0e5ac1e6b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260148863963136","occurred_at":"2026-10-19T19:10:37Z","signature_id":"6271fa840ee018e827576ac0131944df99fffa66b27dc5c1c9b00661db8039a2","pr_number":7497,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x222007964a08\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x222007b84d20\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x2220079649d8\u003e{\n errs: [\n \u003c*errors.errorString | 0x222008c33e60\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x222007964a08\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x222007b84d20\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x2220079649d8\u003e{\n errs: [\n \u003c*errors.errorString | 0x222008c33e60\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","signature_id":"5006c2ff99f83ec65ef1f594a7caa27df5a4f407aeb1491b1d22e62a65fae644","pr_number":7602,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T03:35:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:136]: failed to verify HCP cluster \"cluster-candidate-4-21-d6zdrd\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xde74c1ac0a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xde74c2c7be0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74c2c7bc0\u003e{\n msg: \"failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be74ff0\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xde74bbe7a90\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xde74b4feb80\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74b4feb40\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be1b410\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xde74b874fa0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e0xde74b874e10 (cyclic reference),\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:136]: failed to verify HCP cluster \"cluster-candidate-4-21-d6zdrd\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xde74c1ac0a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xde74c2c7be0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74c2c7bc0\u003e{\n msg: \"failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be74ff0\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xde74bbe7a90\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xde74b4feb80\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74b4feb40\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be1b410\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xde74b874fa0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e0xde74b874e10 (cyclic reference),\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"7573d0a5201f4ebb34194e6054fbfa690d8fbb2ada0cef80ed7ad8a5e9897197","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-15T05:42:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x15ac4447a018\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x15ac43752020\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac44e58300\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac434e29a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x15ac4447a018\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x15ac43752020\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac44e58300\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac434e29a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302759632965632","occurred_at":"2026-10-19T21:59:57Z","signature_id":"303f1c558a1c431ef13a787c38a150317636b1cd97abc6da971b7ee7e7fc947f","pr_number":7613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d74615ba0958ced07d8ca078f31c82f56392409cb82a9380566bf2ff546c6cd4","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x59ce69d41c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x59ce69eea20\u003e{\n s: \"dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x59ce69d41c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x59ce69eea20\u003e{\n s: \"dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112085059526725632","occurred_at":"2026-10-19T07:34:53Z","signature_id":"16703b61dd856f6d7b5d00dca02cd82935f47a24ba99f70f8786b2796812a181","pr_number":6823,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"33372092a33bdf2f33019fa8533fe8a97bd128691c334dc9af187a04e2f74a1c","failure_pattern":"alert [hcp] KubeAPIDown fired","search_query":"alert [hcp] KubeAPIDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeAPIDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeAPIDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:52Z\n Ended: 2026-10-19T15:04:51Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:46:52Z\n Ended: 2026-10-19T15:06:52Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"0a247216608d08991cd5f8a779520976a1aa61b1f0a06325fa95327fe6ee6a06","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b66c7fee67ceccd0ec8a90dea7f05a1d431b6da450c3cf13f7036ae436f2dc62","failure_pattern":"alert [hcp] KubeControllerManagerDown fired","search_query":"alert [hcp] KubeControllerManagerDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeControllerManagerDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeControllerManagerDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:29Z\n Ended: 2026-10-19T15:04:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:47:27Z\n Ended: 2026-10-19T15:07:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"a017e757469163686646b5fba0b071a928dd5160e44056871a323b9c07705567","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-05T11:11:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T11:18:35Z\n Ended: 2026-10-19T11:31:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T11:18:35Z\n Ended: 2026-10-19T11:31:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"1bc65a9f283ab33b10f7701db86d6b6980275fa9553c0469ca59ad241483bdfd","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3774249349c1b01f6f6824e299fdbd82fba2f2355f502a02d66d0fca664aeba8","failure_pattern":"alert [hcp] KubeSchedulerDown fired","search_query":"alert [hcp] KubeSchedulerDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeSchedulerDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeSchedulerDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:54Z\n Ended: 2026-10-19T15:06:57Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:46:54Z\n Ended: 2026-10-19T15:04:55Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"c84bdf2c24901b7b3d2d428c68939f0d21f7d2953227c0a0066c2c9f33f7d7bd","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"272f312e5c168a4a2038538015592572b208afed8fa159d39ec9705fefde4353","failure_pattern":"alert [hcp] KubeletDown fired","search_query":"alert [hcp] KubeletDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeletDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeletDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:14Z\n Ended: 2026-10-19T15:04:13Z\n Severity: Sev2\n Labels: alertname=\"KubeletDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: Kubelet has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:47:17Z\n Ended: 2026-10-19T15:07:16Z\n Severity: Sev2\n Labels: alertname=\"KubeletDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: Kubelet has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"efee231c046f70c9a5514c2cb33f5396255a80ececaf9aa1d9008955e15f6822","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d369dd2a49951034c0c53d59118386a695e898c39ded1ccfd379bf7a8c4f5b5a","failure_pattern":"alert [svc] KubeAPIDown fired","search_query":"alert [svc] KubeAPIDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeAPIDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeAPIDown does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"5670409fa53544e23ab06da52caee5a4a9011a439c224ce3c27e12a7e9ddad05","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a2ba93dd0738827129c2396e0399e755beaa4979871f41d5eda27f307b79c65f","failure_pattern":"alert [svc] KubeControllerManagerDown fired","search_query":"alert [svc] KubeControllerManagerDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeControllerManagerDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeControllerManagerDown does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:28Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"fa8a6cacdaa7a5856fffdfa8d79789f306612470b3ee02cd5790cbaab8b621f1","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-30T16:03:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:56:23Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:56:23Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"ed5cd02b1f495df18e96d006d680e79c415f3cff5710741d4cccbefb8851574d","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":20,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 26 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-19T10:56:21Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-19T11:01:19Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-19T11:01:19Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"2c715e42991a9a3a8d5fdf35df336623aa8ce1ecf9e8c067bf28792e99af1ca0","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-30T16:03:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:57:11Z\n Ended: 2026-10-19T11:07:12Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"prometheus\", node=\"aks-userswft1-30082608-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft1-30082608-vmss000000 is unreachable and some workloads may be rescheduled.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:57:11Z\n Ended: 2026-10-19T11:07:12Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"prometheus\", node=\"aks-userswft1-30082608-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft1-30082608-vmss000000 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"50734fc067690db24537483069cc361f5320a6cc0719aacfbd83716d2a36feaa","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"88b9f437716cb0465d3063d214f7d82310cee5e55f3dc4ff179d878d2ccd19c9","failure_pattern":"alert [svc] KubeSchedulerDown fired","search_query":"alert [svc] KubeSchedulerDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeSchedulerDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeSchedulerDown does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:34Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:35Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:35Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","signature_id":"5d52b99c1caa3445ce1a5a0b42fb7c49f02aae948a373906d50ae2f022e0f597","pr_number":7606,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":19,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":86,"prior_last_seen_at":"2026-10-09T02:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e0c6d55a40\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a20\u003e{\n msg: \"failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a00\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e0c6d55a40\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a20\u003e{\n msg: \"failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a00\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","signature_id":"f9ddbe60cb32d9a3ac38695dbd1398ef4cb5516959103f1774c946569ab2ace1","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-08T16:13:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd2fd780100\u003e: \n failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n {\n msg: \"failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800e0\u003e{\n msg: \"identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800c0\u003e{\n msg: \"get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*azidentity.AuthenticationFailedError | 0x1dd2fee1c150\u003e{\n RawResponse: nil,\n credType: \"ClientSecretCredential\",\n message: \"server response error:\\n context deadline exceeded\",\n omitResponse: false,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd2fd780100\u003e: \n failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n {\n msg: \"failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800e0\u003e{\n msg: \"identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800c0\u003e{\n msg: \"get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*azidentity.AuthenticationFailedError | 0x1dd2fee1c150\u003e{\n RawResponse: nil,\n credType: \"ClientSecretCredential\",\n message: \"server response error:\\n context deadline exceeded\",\n omitResponse: false,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112144183836807168","occurred_at":"2026-10-19T11:29:49Z","signature_id":"78304e8ecc8c474151eda4c7847024ad884d03b5e6f91186cdb7ae5231364c20","pr_number":7065,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"25e28b5e77d38ab357912804c3c34e80b2c640b1e778e31291607291be7016d6","failure_pattern":"expected \u003ccount\u003e ready (and schedulable) nodes, found \u003ccount\u003e; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [\u003cnode\u003e, \u003cnode\u003e]","search_query":"expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:373]: expected 2 ready and schedulable nodes after the cordon/drain/uncordon cycle\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3f000f2687f0\u003e: \n expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\n {\n s: \"expected 2 ready (and schedulable) nodes in node pool \\\"cordon-np\\\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:373]: expected 2 ready and schedulable nodes after the cordon/drain/uncordon cycle\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3f000f2687f0\u003e: \n expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\n {\n s: \"expected 2 ready (and schedulable) nodes in node pool \\\"cordon-np\\\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","signature_id":"95ce875d62f45d254fefee15188f37e24bc247d864e7512f855578248bd38bb4","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,1,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-18T21:21:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1}],"full_error_samples":["time=2026-10-19T15:30:44.609Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-19T15:30:46.080Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-19T15:30:46.080Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-19T15:30:46.888Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-19T15:30:46.889Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-19T15:30:46.889Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8401536-svc\ntime=2026-10-19T15:45:50.026Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7506/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112197072718401536","occurred_at":"2026-10-19T14:59:44Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7506,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.005s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.005s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","signature_id":"ae7f359019eb8ec65c1a72a87e13d829e08e16887d3280b98246e0e23d7fa164","pr_number":7597,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-16T10:29:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1c5cb893a690\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-h758j8sh8qhg\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1c5cb889c060\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1c5cb893a690\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-h758j8sh8qhg\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1c5cb889c060\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","signature_id":"17a4d563dd3fd1d5f96570c753ad16a1d18bf9d1bdf8b5b3ea44580c362c8768","pr_number":7582,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0869565217391304,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[2,26,18,10,1,0,1],"trend_range":"2026-10-13..2026-10-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":58,"prior_last_seen_at":"2026-10-17T21:48:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cde6af4c2e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1cde6af4c2c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1cde6a0b6140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1cde6af4c2a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1cde6c048750\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cde6af4c2e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1cde6af4c2c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1cde6a0b6140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1cde6af4c2a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1cde6c048750\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","signature_id":"386c6869d89c07e4c74c5f01b6e960ca1675fb507e88a5f58cb3f2db3a845593","pr_number":7232,"after_last_push_of_merged_pr":false}]}]}]}