{"meta":{"start_date":"2026-10-01","end_date":"2026-10-01","start_at":"2026-10-01T00:00:00Z","end_at":"2026-10-02T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T09:58:57Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":86,"failed_runs":53,"raw_occurrences":85,"matched_occurrences":75,"runs_affected":43},"rows":[{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":21,"runs_affected":16,"failed_runs":16,"run_impact_percent":18.6046511627907,"anchor_occurrences":21,"after_last_push_count":24,"also_in":["INT","PROD","STG"],"trend_counts":[6,2,7,19,10,8,21],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":63,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013bd140\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xj578sgpwt6l\" to finish creating: GET https://rp.j2660096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0fba71ef-f7ca-46bc-a499-4c95fff82d8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0fba71ef-f7ca-46bc-a499-4c95fff82d8b\",\n      \"name\": \"0fba71ef-f7ca-46bc-a499-4c95fff82d8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:27:55.227836447Z\",\n      \"endTime\": \"2026-10-02T00:47:04.606186902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-xj578sgpwt6l--managed/providers/Microsoft.Compute/virtualMachines/u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-xj578sgpwt6l--managed/providers/Microsoft.Compute/virtualMachines/u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013bd140\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xj578sgpwt6l\" to finish creating: GET https://rp.j2660096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0fba71ef-f7ca-46bc-a499-4c95fff82d8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0fba71ef-f7ca-46bc-a499-4c95fff82d8b\",\n      \"name\": \"0fba71ef-f7ca-46bc-a499-4c95fff82d8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:27:55.227836447Z\",\n      \"endTime\": \"2026-10-02T00:47:04.606186902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-xj578sgpwt6l--managed/providers/Microsoft.Compute/virtualMachines/u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-xj578sgpwt6l--managed/providers/Microsoft.Compute/virtualMachines/u9v6s2e4a2b5l3l-autoscale-noaz-f87v6-bwdd7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:182]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb92c0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-d4n779\" in resourcegroup=\"rg-np-version-upgrade-d4n779-msk5sf2n8dpt\" to finish creating: GET https://rp.j8032640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d667a581-6512-4aad-a4a8-0592dbc7103f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d667a581-6512-4aad-a4a8-0592dbc7103f\",\n      \"name\": \"d667a581-6512-4aad-a4a8-0592dbc7103f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:18:31.430555435Z\",\n      \"endTime\": \"2026-10-02T00:37:32.349627981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-d4n779-msk5sf2n8dpt-np-upgrade-d4-ed3dd0ec/providers/Microsoft.Compute/virtualMachines/l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-d4n779-msk5sf2n8dpt-np-upgrade-d4-ed3dd0ec/providers/Microsoft.Compute/virtualMachines/l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:182]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb92c0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-d4n779\" in resourcegroup=\"rg-np-version-upgrade-d4n779-msk5sf2n8dpt\" to finish creating: GET https://rp.j8032640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d667a581-6512-4aad-a4a8-0592dbc7103f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d667a581-6512-4aad-a4a8-0592dbc7103f\",\n      \"name\": \"d667a581-6512-4aad-a4a8-0592dbc7103f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:18:31.430555435Z\",\n      \"endTime\": \"2026-10-02T00:37:32.349627981Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-d4n779-msk5sf2n8dpt-np-upgrade-d4-ed3dd0ec/providers/Microsoft.Compute/virtualMachines/l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-d4n779-msk5sf2n8dpt-np-upgrade-d4-ed3dd0ec/providers/Microsoft.Compute/virtualMachines/l0u3m1v2v6n0r3x-npupgrade-4-21-bvg9t-b9ms7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-xxkkqq\" for cluster \"cluster-hs-pre-xxkkqq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a7280\u003e: \n    failed to create NodePool np-xxkkqq: failed waiting for nodepool=\"np-xxkkqq\" for cluster \"cluster-hs-pre-xxkkqq\" in resourcegroup=\"rg-hs-pre--9kx2vcffdvgg\" to finish creating: GET https://rp.j0967040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b6a2e90-815b-425d-ab8b-0de7336ee353\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b6a2e90-815b-425d-ab8b-0de7336ee353\",\n      \"name\": \"9b6a2e90-815b-425d-ab8b-0de7336ee353\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T22:20:04.161467493Z\",\n      \"endTime\": \"2026-10-01T22:39:14.657703099Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--9kx2vcffdvgg-xxkkqq--managed/providers/Microsoft.Compute/virtualMachines/h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--9kx2vcffdvgg-xxkkqq--managed/providers/Microsoft.Compute/virtualMachines/h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-xxkkqq\" for cluster \"cluster-hs-pre-xxkkqq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a7280\u003e: \n    failed to create NodePool np-xxkkqq: failed waiting for nodepool=\"np-xxkkqq\" for cluster \"cluster-hs-pre-xxkkqq\" in resourcegroup=\"rg-hs-pre--9kx2vcffdvgg\" to finish creating: GET https://rp.j0967040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b6a2e90-815b-425d-ab8b-0de7336ee353\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b6a2e90-815b-425d-ab8b-0de7336ee353\",\n      \"name\": \"9b6a2e90-815b-425d-ab8b-0de7336ee353\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T22:20:04.161467493Z\",\n      \"endTime\": \"2026-10-01T22:39:14.657703099Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--9kx2vcffdvgg-xxkkqq--managed/providers/Microsoft.Compute/virtualMachines/h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--9kx2vcffdvgg-xxkkqq--managed/providers/Microsoft.Compute/virtualMachines/h3e6a3z6w8m5h2c-np-xxkkqq-2gbgl-78tdt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105799429062660096","occurred_at":"2026-10-01T23:18:02Z","signature_id":"15c9cc6560f8e49e5e0d171464146a487c2ce0945c7923982699a3ee52216d09","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105791127218032640","occurred_at":"2026-10-01T22:45:03Z","signature_id":"3ab7ed9ca9460bb22a2cdd805c04ae36e791c3168b7f3b5c56679dc030b8b973","pr_number":7293,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105766044650967040","occurred_at":"2026-10-01T21:05:22Z","signature_id":"9d61e03aed47d95b1496ea6b301d3b295e38f116b12cfc5bd827220d9051fc5a","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105766044650967040","occurred_at":"2026-10-01T21:05:22Z","signature_id":"d53a72a8f56dff0d24e120d19311677914cc388704d8109e5516520aaa51b2c9","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105721100032806912","occurred_at":"2026-10-01T18:06:47Z","signature_id":"14ec8474b0adde86a2d56a6739e8d72786179111dfad63509ea9fd6c4db44447","pr_number":7303,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7296/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105705639958286336","occurred_at":"2026-10-01T17:05:19Z","signature_id":"75682bb326c7e07d9b5624939e163e4b1ec841db6f93a368fec9b822dcdbd2ac","pr_number":7296,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105700267935666176","occurred_at":"2026-10-01T16:44:00Z","signature_id":"6a12831993ef9e949e6833ef6f25d132163da8f4ed2625b08c2474daaae3de21","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105700267935666176","occurred_at":"2026-10-01T16:44:00Z","signature_id":"add74e624f5642c937662cf1bfc782a00394e37e6a637d085846e992fa545679","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105693768928727040","occurred_at":"2026-10-01T16:18:10Z","signature_id":"4d3f102a2b9e95e636a1d28e7d89a2468478ccfc93c9ec0c0a407b2d51fbec62","pr_number":7227,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105693768928727040","occurred_at":"2026-10-01T16:18:10Z","signature_id":"a91def54ce68cf33e933c8f76532bd3ec0f0a1820e97c58b2ca1df1f44b4483d","pr_number":7227,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105656281044881408","occurred_at":"2026-10-01T13:49:13Z","signature_id":"271fee169ddf537a30ccb8e914b976f8a128cfb6b0fc16b307d0929759f67821","pr_number":6775,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105647499015360512","occurred_at":"2026-10-01T13:14:18Z","signature_id":"5a8b630f838fcf686fe9ea13df4dd4e3b52d4b36e14b45811335b1ecfb2cd6a4","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105619693527109632","occurred_at":"2026-10-01T11:23:50Z","signature_id":"e8796b6cd7f96bce574b45c34f4741d0ec53224a8a314ad86fa79ba998b1dcb8","pr_number":7112,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105609202608115712","occurred_at":"2026-10-01T10:42:08Z","signature_id":"34f4af77ab97bc88e0f7d941ba1af35727fc299abf9e49a9dee6df9873f8aa24","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105608440268197888","occurred_at":"2026-10-01T10:39:07Z","signature_id":"951fca799dcb27322e5db5158692fd9c1afb506faaa4a3276c21396694bac07f","pr_number":7276,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105598359803793408","occurred_at":"2026-10-01T09:59:03Z","signature_id":"871377b9a0c51357a883e114688e7fa2e5c5eb73473defc7c22fe2f98103f6d1","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105598359803793408","occurred_at":"2026-10-01T09:59:03Z","signature_id":"9a97450be9bc55853dbc20b75db0bc18e91f72e8281ae92028c362ae14c6ecbd","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105596656899592192","occurred_at":"2026-10-01T09:52:17Z","signature_id":"9eb76b31f35a0f3355fe6d1fd541747b42aa9bd645ba4215e2cb626458d66625","pr_number":7268,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105594300950646784","occurred_at":"2026-10-01T09:42:56Z","signature_id":"88e6d923667a18f63bb5d9d679c24042ddb61662863d7416a3373acde9b7a86f","pr_number":7266,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105594300950646784","occurred_at":"2026-10-01T09:42:56Z","signature_id":"f3506ffc9a8fd4b13ae19da2228f803a1c6714ae4bdb67259ba8b79387eeb870","pr_number":7266,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105572392540573696","occurred_at":"2026-10-01T08:15:43Z","signature_id":"cbe0e7c1fea8f37a2e2c48ddaae5fe0f22a5fe82949c5c5ed01d6db24002412d","pr_number":7276,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":6.976744186046512,"anchor_occurrences":6,"after_last_push_count":63,"trend_counts":[6,1,2,1,7,10,6],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":127,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T23:46:43Z\n  Ended:       2026-10-01T23:52:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1600128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t69mr53lpaagaj8ndpfbdmi4pj01tkd-j7t7p3j9q1q0x2g\", pod=\"machine-approver-74799c449d-qwlv2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t69mr53lpaagaj8ndpfbdmi4pj01tkd-j7t7p3j9q1q0x2g/machine-approver-74799c449d-qwlv2 has been in a non-ready state for longer than 5 minutes.","alert fired 96 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-mwtcx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-mwtcx has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-vvwct\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-vvwct has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-7bbsw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-7bbsw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-v4ftx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-v4ftx has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-zgtdx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-zgtdx has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-42mfl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-42mfl has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-lvnnt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-lvnnt has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-pghsk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-pghsk has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-gxkml\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-gxkml has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-jlzrr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-jlzrr has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-lvjnb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-lvjnb has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-4jj9w\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-4jj9w has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-fpstv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-fpstv has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-cgnq6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-cgnq6 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-gxr4d\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-gxr4d has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-98xtm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-98xtm has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-zc7gh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-zc7gh has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-qjtjc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-qjtjc has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-7dl97\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-7dl97 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-msjxj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-msjxj has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-lcvdj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-lcvdj has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-cbjnh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-cbjnh has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-wbsrk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-wbsrk has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-88hwc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-88hwc has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-vpc4x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-vpc4x has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-rm7bz\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-rm7bz has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-frwsn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-frwsn has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-bmpv2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-bmpv2 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-psq6v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-psq6v has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-59fkb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-59fkb has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-s9hxt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-s9hxt has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-d5ggn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-d5ggn has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-mz8hk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-mz8hk has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-rps7j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-rps7j has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-gf894\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-gf894 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-9zvzb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-9zvzb has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-29d68\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-29d68 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-g8db9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-g8db9 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-gn9th\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-gn9th has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-4pklx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-4pklx has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-f8zbt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-f8zbt has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-qlxtj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-qlxtj has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-zrnlh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-zrnlh has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-6m7wq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-6m7wq has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-9rhtk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-9rhtk has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-bl4jg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-bl4jg has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-8pmcr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-8pmcr has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-9nxwc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-9nxwc has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-9f96s\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-9f96s has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-mmp68\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-mmp68 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-d4nqn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-d4nqn has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-c5nnh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-c5nnh has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-t48z7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-t48z7 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-q4bwx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-q4bwx has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:30:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-f9m5n\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-f9m5n has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-xb9s5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-xb9s5 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-jh25x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-jh25x has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n  State:       Resolved\n  Started:     2026-10-01T18:07:41Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-g9tw9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-g9tw9 has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n  State:       Resolved\n  Started:     2026-10-01T18:08:38Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-r6qng\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-r6qng has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n  State:       Resolved\n  Started:     2026-10-01T18:08:38Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-74wcq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-74wcq has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n  State:       Resolved\n  Started:     2026-10-01T18:08:38Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-7lct6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-7lct6 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n  State:       Resolved\n  Started:     2026-10-01T18:08:38Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-zkbkk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-zkbkk has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-v7m2p\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-v7m2p has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-dh2wj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-dh2wj has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-9frwj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-9frwj has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-bvxz5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-bvxz5 has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-hfm7f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-hfm7f has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-wjp8n\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-wjp8n has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-n5vmn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-n5vmn has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-d8lx7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-d8lx7 has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-dlhhb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-dlhhb has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-k69bw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-k69bw has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-kczlx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-kczlx has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-zdgk7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-zdgk7 has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-rvmlf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-rvmlf has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-gmfc6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-gmfc6 has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-lddj8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-lddj8 has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-xm6fg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-xm6fg has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-fn46n\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-fn46n has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-58r86\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-58r86 has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-cfqmp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-cfqmp has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-cpccl\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-cpccl has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-th4tc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-th4tc has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-2xsng\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-2xsng has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-z8ldz\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-z8ldz has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-96jbc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-96jbc has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-cltv6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-cltv6 has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-sfd6l\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-sfd6l has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-bgmnc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-bgmnc has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-phv2m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-phv2m has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-2jlkd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-2jlkd has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:33:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-cf2v5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-cf2v5 has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-gmspn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-gmspn has been in a non-ready state for longer than 5 minutes.\nFiring 94:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-7rkp9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-7rkp9 has been in a non-ready state for longer than 5 minutes.\nFiring 95:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:28:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y\", pod=\"ignition-server-proxy-7fb844bf7d-2566w\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v6276f370teuh06rg23hq77p5h0s-l2e2a6g2r9h9h0y/ignition-server-proxy-7fb844bf7d-2566w has been in a non-ready state for longer than 5 minutes.\nFiring 96:\n  State:       Resolved\n  Started:     2026-10-01T18:08:39Z\n  Ended:       2026-10-01T18:30:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5666176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f\", pod=\"kube-controller-manager-7cfc7cc4bb-66jst\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t64v3u9bus0muojo3trqcut8bp50jk4-v8l6m9c7b8s7p9f/kube-controller-manager-7cfc7cc4bb-66jst has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T12:11:06Z\n  Ended:       2026-10-01T12:17:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8197888-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5vg8dltnlcbk8c17tn7tb3mpjhgjq9-u5u0k2w2t5m6a1t\", pod=\"openshift-apiserver-58f5558dc-lmkqq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5vg8dltnlcbk8c17tn7tb3mpjhgjq9-u5u0k2w2t5m6a1t/openshift-apiserver-58f5558dc-lmkqq has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105783166311600128","occurred_at":"2026-10-01T22:13:25Z","signature_id":"ec70362871ff363be2ba6fd5534174a58e4198efdec597628c3b037bc28e9c44","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105700267935666176","occurred_at":"2026-10-01T16:44:00Z","signature_id":"e18d9d0c14ea1e1c9f30500c22e3f8f0572012f806138af0f3a6ad156c3555fc","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105608440268197888","occurred_at":"2026-10-01T10:39:07Z","signature_id":"c22f9abc449f666441524c22c9f34791a2317794ac1154d80c7795ae5181427b","pr_number":7276,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105594300950646784","occurred_at":"2026-10-01T09:42:56Z","signature_id":"dbd2e67aa2d46fb2c08567dcda109d64707d55b3dfd554e6ea87eb76d61bdc98","pr_number":7266,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105536022291943424","occurred_at":"2026-10-01T05:51:21Z","signature_id":"fa70e46ed99644e477bea4e1b8d44c494f7d08a27f406bf2f556fa3421e5fd06","pr_number":7283,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105460820702924800","occurred_at":"2026-10-01T00:52:31Z","signature_id":"665ebec96b10a377b87be5e9fb862b38f6f08cfaf7afbd04a8455116db4e844f","pr_number":7276,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.813953488372093,"anchor_occurrences":5,"after_last_push_count":3,"trend_counts":[2,1,8,1,19,3,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":18,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001422540\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nr76gskcfsht\" to finish creating: GET https://rp.j8617344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\",\n      \"name\": \"d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T21:37:37.328442226Z\",\n      \"endTime\": \"2026-10-01T22:06:41.36782171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001422540\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nr76gskcfsht\" to finish creating: GET https://rp.j8617344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\",\n      \"name\": \"d63268b6-f286-4ac7-9dcb-f75b1fa2b29a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T21:37:37.328442226Z\",\n      \"endTime\": \"2026-10-01T22:06:41.36782171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-dhvd4s\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed0460\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-dhvd4s: failed waiting for cluster=\"cp-ystream-upgrade-5-1-dhvd4s\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-25w79q54vqwd\" to finish creating: GET https://rp.j4153600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4f26d35-7dcc-4a11-95e9-ba0b44e92249\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4f26d35-7dcc-4a11-95e9-ba0b44e92249\",\n      \"name\": \"e4f26d35-7dcc-4a11-95e9-ba0b44e92249\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:01:36.572713358Z\",\n      \"endTime\": \"2026-10-01T18:30:44.899304602Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-dhvd4s\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed0460\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-dhvd4s: failed waiting for cluster=\"cp-ystream-upgrade-5-1-dhvd4s\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-25w79q54vqwd\" to finish creating: GET https://rp.j4153600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4f26d35-7dcc-4a11-95e9-ba0b44e92249\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4f26d35-7dcc-4a11-95e9-ba0b44e92249\",\n      \"name\": \"e4f26d35-7dcc-4a11-95e9-ba0b44e92249\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:01:36.572713358Z\",\n      \"endTime\": \"2026-10-01T18:30:44.899304602Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-jsgjhmrsczb4/cluster-candidate-4-23-tqz4r8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a41c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-tqz4r8: failed waiting for cluster=\"cluster-candidate-4-23-tqz4r8\" in resourcegroup=\"rg-candidate-4-23-jsgjhmrsczb4\" to finish creating: GET https://rp.j9824640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b3b1244-2be2-4c28-a2e6-54d138ae923b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b3b1244-2be2-4c28-a2e6-54d138ae923b\",\n      \"name\": \"3b3b1244-2be2-4c28-a2e6-54d138ae923b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T11:58:08.343305291Z\",\n      \"endTime\": \"2026-10-01T12:27:12.911892104Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-jsgjhmrsczb4/cluster-candidate-4-23-tqz4r8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a41c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-tqz4r8: failed waiting for cluster=\"cluster-candidate-4-23-tqz4r8\" in resourcegroup=\"rg-candidate-4-23-jsgjhmrsczb4\" to finish creating: GET https://rp.j9824640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b3b1244-2be2-4c28-a2e6-54d138ae923b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b3b1244-2be2-4c28-a2e6-54d138ae923b\",\n      \"name\": \"3b3b1244-2be2-4c28-a2e6-54d138ae923b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T11:58:08.343305291Z\",\n      \"endTime\": \"2026-10-01T12:27:12.911892104Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105758317698617344","occurred_at":"2026-10-01T20:34:40Z","signature_id":"c986760a5f4d2652dc43b9dabc7fd3c14d4d7b36fd44e885ee97cbd0025fa91c","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105707780064153600","occurred_at":"2026-10-01T17:13:51Z","signature_id":"a16221c27ccc95fd81b9b66186bd1e94cffc7a154233bcad87a898d3ca7bc7aa","pr_number":6900,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105614591059824640","occurred_at":"2026-10-01T11:03:33Z","signature_id":"847608b38969ba5c7930f2ebf2a8e80aa7fc90c5350fca4079d02a6659c6376c","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105599650848313344","occurred_at":"2026-10-01T10:04:11Z","signature_id":"6cad2bd1182716f198eae4d3ca1302694e3844a304229000581b0c5cee1867c4","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105596839066603520","occurred_at":"2026-10-01T09:53:01Z","signature_id":"af3af6131995b25e21b59d0019ab66075ba5903968947ddea08ec1f76bab3241","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.813953488372093,"anchor_occurrences":5,"after_last_push_count":15,"trend_counts":[2,1,0,5,3,4,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":45,"prior_last_seen_at":"2026-09-26T19:18:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T22:11:26Z\n  Ended:       2026-10-01T22:26:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gfv2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T22:11:26Z\n  Ended:       2026-10-01T22:26:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gfv2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T18:36:27Z\n  Ended:       2026-10-01T18:50:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4153600-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4153600/providers/microsoft.monitor/accounts/hcps-j4153600\", namespace=\"ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qpxgq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T12:32:56Z\n  Ended:       2026-10-01T12:47:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mrs5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T12:32:56Z\n  Ended:       2026-10-01T12:46:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mrs5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105758317698617344","occurred_at":"2026-10-01T20:34:40Z","signature_id":"29d673db59c2b087d66ae8fe2bdccf9959159f2ce656d055e6c807a3ccf4d8fc","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105707780064153600","occurred_at":"2026-10-01T17:13:51Z","signature_id":"57ca4cce7b8ccf5ef5108d8b8575fadb54d7ed7b223e0d0d06bdd9e11973fecd","pr_number":6900,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105614591059824640","occurred_at":"2026-10-01T11:03:33Z","signature_id":"dcd1274ba237d9ec50df939e07a72cd403c9be3e0e7cc9f93ed5ff4a4f75df03","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105599650848313344","occurred_at":"2026-10-01T10:04:11Z","signature_id":"58bb9ab98c2001634ef201c0fedb959cb8482213626f7bc765d28d5bcb5d2877","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105596839066603520","occurred_at":"2026-10-01T09:53:01Z","signature_id":"14833ce9f312de06b64c36f8da279553e96a2229f9d94f0bca0ebbdb97fcb5e4","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.813953488372093,"anchor_occurrences":5,"after_last_push_count":12,"trend_counts":[1,1,0,1,2,3,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":26,"prior_last_seen_at":"2026-09-26T22:54:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T21:59:25Z\n  Ended:       2026-10-01T22:29:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a69f5d7-6f9f-4450-a9f7-f182ad9a23c6\"\n  Description: Pod ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T21:59:25Z\n  Ended:       2026-10-01T22:29:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0a69f5d7-6f9f-4450-a9f7-f182ad9a23c6\"\n  Description: Pod ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T18:23:21Z\n  Ended:       2026-10-01T18:53:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4153600-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4153600/providers/microsoft.monitor/accounts/hcps-j4153600\", namespace=\"ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"71bdc953-4c3d-4f04-a433-73b33d9edd12\"\n  Description: Pod ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T18:23:21Z\n  Ended:       2026-10-01T18:53:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4153600-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4153600/providers/microsoft.monitor/accounts/hcps-j4153600\", namespace=\"ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"71bdc953-4c3d-4f04-a433-73b33d9edd12\"\n  Description: Pod ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T12:20:58Z\n  Ended:       2026-10-01T12:50:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f591f195-5e0c-485f-a44a-f0bfd5b23bc7\"\n  Description: Pod ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T12:20:58Z\n  Ended:       2026-10-01T12:49:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f591f195-5e0c-485f-a44a-f0bfd5b23bc7\"\n  Description: Pod ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105758317698617344","occurred_at":"2026-10-01T20:34:40Z","signature_id":"3f634f10a7d0ff56a207967252f4dd779dc50e1538b6810f21ad2cb7a80518a8","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105707780064153600","occurred_at":"2026-10-01T17:13:51Z","signature_id":"2bbb003123b12a1a01833a71d68db9c0fd3861c32c0fab1d187778af335a84dc","pr_number":6900,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105614591059824640","occurred_at":"2026-10-01T11:03:33Z","signature_id":"33a95e1e5860ff5fbe825e8fb85c950568c6df5470a1a6ee4b7714eef0d17220","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105599650848313344","occurred_at":"2026-10-01T10:04:11Z","signature_id":"e17a233183bb4644662713f3770bc46600a5a1d07157ebd0249bb8d0eb316e5c","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105596839066603520","occurred_at":"2026-10-01T09:53:01Z","signature_id":"d795eb80a92c1fc3c9bb19941bbe4c3e4082f2f869b33c35f3d75c6f2b5ca225","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.813953488372093,"anchor_occurrences":5,"after_last_push_count":20,"trend_counts":[3,1,1,1,5,2,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":43,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T21:56:38Z\n  Ended:       2026-10-01T22:24:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gfv2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T21:56:38Z\n  Ended:       2026-10-01T22:24:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8617344-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8617344/providers/microsoft.monitor/accounts/hcps-j8617344\", namespace=\"ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gfv2r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t6898csqijj9hrgrrv2thav2ags7h5e-u0x9h1i4c5m9r3t/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T18:21:23Z\n  Ended:       2026-10-01T18:49:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4153600-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4153600/providers/microsoft.monitor/accounts/hcps-j4153600\", namespace=\"ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qpxgq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T18:32:21Z\n  Ended:       2026-10-01T18:49:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4153600-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4153600/providers/microsoft.monitor/accounts/hcps-j4153600\", namespace=\"ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qpxgq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t6540sfbs5tnji617qi625vcboi5s6g-v2e1p4r9r2e9x7a/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T12:19:00Z\n  Ended:       2026-10-01T12:45:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mrs5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T12:19:00Z\n  Ended:       2026-10-01T12:45:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9824640-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9824640/providers/microsoft.monitor/accounts/hcps-j9824640\", namespace=\"ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mrs5f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5vpkfr8mo8mfg13tg03gjq59s5mg0f-d4e3m6p6t4b1r3d/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6924/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105758317698617344","occurred_at":"2026-10-01T20:34:40Z","signature_id":"49835a63b5eeb6cd2d5676b383b748e801852cbfc4a14f2ccad62f35c1379e35","pr_number":6924,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105707780064153600","occurred_at":"2026-10-01T17:13:51Z","signature_id":"42a4652e36610b982e285e479e664235522917c24849effb4d93491c383305f6","pr_number":6900,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105614591059824640","occurred_at":"2026-10-01T11:03:33Z","signature_id":"4427a7f0cd67abf2bad6097b35ebbeb0f0f68f33f229f36a2cbb2241ba70ce75","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105599650848313344","occurred_at":"2026-10-01T10:04:11Z","signature_id":"401a1e204a83fda3675c86a680e28f32700788a8e903ffd959925db946223e78","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105596839066603520","occurred_at":"2026-10-01T09:53:01Z","signature_id":"3db0b13f6503d0d391e7cd47b49850d3afd397f715312359fe93c62841f3e4e5","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":5.813953488372093,"anchor_occurrences":5,"after_last_push_count":23,"trend_counts":[12,0,2,7,11,3,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":65,"prior_last_seen_at":"2026-09-27T13:27:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":5}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T23:10:36Z\n  Ended:       2026-10-01T23:23:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9983616-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-01T23:11:38Z\n  Ended:       2026-10-01T23:25:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9983616-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-01T23:12:38Z\n  Ended:       2026-10-01T23:23:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9983616-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T14:26:18Z\n  Ended:       2026-10-01T14:44:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4147456-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T11:09:02Z\n  Ended:       2026-10-01T11:15:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8313344-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105779586879983616","occurred_at":"2026-10-01T21:59:11Z","signature_id":"4126cd70f9e258007e4e457356ba9c9a093cd31c782620f814f33f5d61f037f4","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105647975744147456","occurred_at":"2026-10-01T13:16:12Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7276,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105599650848313344","occurred_at":"2026-10-01T10:04:11Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105473667151433728","occurred_at":"2026-10-01T01:43:34Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7181,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105468689062039552","occurred_at":"2026-10-01T01:23:47Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7272,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.488372093023256,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,1,2,2,3],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-21T22:11:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":3}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T19:40:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2640384-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.90:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2640384/providers/microsoft.monitor/accounts/services-j2640384\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-77d869db9f-wr2wh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2640384-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T19:40:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2640384-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.90:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2640384/providers/microsoft.monitor/accounts/services-j2640384\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-77d869db9f-wr2wh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2640384-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T15:34:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4147456-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4147456/providers/microsoft.monitor/accounts/services-j4147456\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-9976f6db4-kdw9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j4147456-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T15:35:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4147456-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4147456/providers/microsoft.monitor/accounts/services-j4147456\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-9976f6db4-kdw9q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j4147456-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T14:24:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j5027584-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5027584/providers/microsoft.monitor/accounts/services-j5027584\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-65fddd48b7-ckqvd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j5027584-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T14:24:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j5027584-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5027584/providers/microsoft.monitor/accounts/services-j5027584\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-65fddd48b7-ckqvd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j5027584-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105711608922640384","occurred_at":"2026-10-01T17:29:03Z","signature_id":"9611594927e244c91913b4314033ab5b63beb5f07a3d7300efcb35da238063ea","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105647975744147456","occurred_at":"2026-10-01T13:16:12Z","signature_id":"200970a28019241ac966ca5f4a5c15f93b321fc381704802b4f75d7ffe63a64e","pr_number":7276,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7055/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105632268105027584","occurred_at":"2026-10-01T12:13:48Z","signature_id":"46285051710f82d9ccbacfb63d65d4927b7dd5f26a64b6f33d16b4152ae9fa45","pr_number":7055,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8af94ff77be9280ec3040c21d82445dd604d4b00fdf26e371d68448461bff48d","failure_pattern":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-\u003cid\u003e/pipeline@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/ci-op-\u003cid\u003e/pipeline:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-49jbb95d/pipeline@sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 arohcpsvcdev.azurecr.io/ci-op-49jbb95d/pipeline:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 --from-registry-config /tmp/tmp.WQPJJsgkPI/containers/auth.json --to-registry-config /tmp/tmp.WQPJJsgkPI/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.3255813953488373,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.MgmtAgent/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-10-01T15:34:21.055Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.MgmtAgent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e mgmtAgent.image.registry \u003cnil\u003e}:{\u003cnil\u003e mgmtAgent.image.repository \u003cnil\u003e}@{\u003cnil\u003e mgmtAgent.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-01T15:35:58.894Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.MgmtAgent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.RKxD35P2Nz/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-8mw8v43n/pipeline@sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b to arohcpsvcdev.azurecr.io/ci-op-8mw8v43n/pipeline:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b.\\nThe image will still be available under it's original digest sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b in the target registry.\\nError response from registry: failed to resolve sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-8mw8v43n/pipeline@sha256:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b arohcpsvcdev.azurecr.io/ci-op-8mw8v43n/pipeline:8d86cc812124ab7af5c3f581f9f2865d36cb1fd9340ecc28481c6d525c0cfa0b --from-registry-config /tmp/tmp.RKxD35P2Nz/containers/auth.json --to-registry-config /tmp/tmp.RKxD35P2Nz/containers/auth.json\\n exit status 1\"","time=2026-10-01T15:33:44.701Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e fleet.image.registry \u003cnil\u003e}:{\u003cnil\u003e fleet.image.repository \u003cnil\u003e}@{\u003cnil\u003e fleet.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-01T15:35:32.216Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.WQPJJsgkPI/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-49jbb95d/pipeline@sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 to arohcpsvcdev.azurecr.io/ci-op-49jbb95d/pipeline:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14.\\nThe image will still be available under it's original digest sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 in the target registry.\\nError response from registry: failed to resolve sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-49jbb95d/pipeline@sha256:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 arohcpsvcdev.azurecr.io/ci-op-49jbb95d/pipeline:0d10cd3c8fa8572aa7784bf13bac7b0907199a7c1851db08138b412e3f25df14 --from-registry-config /tmp/tmp.WQPJJsgkPI/containers/auth.json --to-registry-config /tmp/tmp.WQPJJsgkPI/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105670637572853760","occurred_at":"2026-10-01T14:46:16Z","signature_id":"94ee0cea4a4ee0ca9f3baeefcfcae22623fa46b795ccfa005026a3378ffdef00","pr_number":6900,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105670047690133504","occurred_at":"2026-10-01T14:43:55Z","signature_id":"2d70df3f3655f8e6d61803842ff1a711d8b43be234f53bc324901afc6271aab3","pr_number":7281,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.3255813953488373,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,3],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T11:48:27Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-10-01T19:47:54.330Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-01T19:47:56.434Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-01T19:47:57.612Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=7fa4e9b0d5432f3733a8410e6c9eae279686c4d2104771439ec6c49754f75d3c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2785792-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7fa4e9b0d5432f3733a8410e6c9eae279686c4d2104771439ec6c49754f75d3c\ntime=2026-10-01T19:50:58.074Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2785792-svc/providers/Microsoft.Resources/deployments/7fa4e9b0d5432f3733a8410e6c9eae279686c4d2104771439ec6c49754f75d3c/operationStatuses/08584107228086173802\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2785792-svc/providers/Microsoft.Resources/deployments/cluster-suocoh45plnvg\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2785792-svc/providers/Microsoft.Resources/deployments/ah-ci01-se-j2785792-1\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2785792-svc/providers/Microsoft.Resources/deployments/ah-ci01-se-j2785792-1\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-se-j2785792-1' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-10-01T19:47:48.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-01T19:47:49.512Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-01T19:47:50.952Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=5be4ec9a5f0977ac9396f709d63fcfad01e98a314c6ed4279d81457cdf4fb342 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0582272-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5be4ec9a5f0977ac9396f709d63fcfad01e98a314c6ed4279d81457cdf4fb342\ntime=2026-10-01T19:50:21.739Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/5be4ec9a5f0977ac9396f709d63fcfad01e98a314c6ed4279d81457cdf4fb342/operationStatuses/08584107228152517010\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0582272-2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/mgmt-kv-2awbxe73l63vi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mg-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/msi-kv-g2p3wnorghzau\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-mi-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7306/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105741109882785792","occurred_at":"2026-10-01T19:26:17Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":7306,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105738861190582272","occurred_at":"2026-10-01T19:17:21Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-mbclvmsdh2bf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-mbclvmsdh2bf, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.3255813953488373,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":21,"prior_last_seen_at":"2026-09-27T08:18:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013de390\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-vvt8wh\" resourcegroup=\"rg-np-version-upgrade-vvt8wh-bgjbrzltjmjv\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-vvt8wh in resource group rg-np-version-upgrade-vvt8wh-bgjbrzltjmjv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013de390\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-vvt8wh\" resourcegroup=\"rg-np-version-upgrade-vvt8wh-bgjbrzltjmjv\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-vvt8wh in resource group rg-np-version-upgrade-vvt8wh-bgjbrzltjmjv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002898f60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-mbclvmsdh2bf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-mbclvmsdh2bf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002898f60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-mbclvmsdh2bf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-mbclvmsdh2bf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105647975744147456","occurred_at":"2026-10-01T13:16:12Z","signature_id":"ed1a46495fda273c61529fbe806961a34166aee46518c8ec3ba7c537be09f530","pr_number":7276,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105612041917042688","occurred_at":"2026-10-01T10:53:25Z","signature_id":"de501cbd0eabfd299a0e7583cc7f6e70873b149915b0f4ca427f669d7e6529c0","pr_number":7288,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.3255813953488373,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:182]: Timed out after 1200.000s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:180 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: ...\nto be \u003c\n    \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:182]: Timed out after 1200.000s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:180 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: ...\nto be \u003c\n    \u003cint\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:182]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:180 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: ...\nto be \u003c\n    \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:182]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:180 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: ...\nto be \u003c\n    \u003cint\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105779586879983616","occurred_at":"2026-10-01T21:59:11Z","signature_id":"16d6edf02807e62fdd39eb4599707ae45d2ddda910e7117f103d5b186415d662","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105728864041832448","occurred_at":"2026-10-01T18:37:38Z","signature_id":"35d1a89e22e4294e225357121ab3f545c078326bf4bf5c105c3f49437b9f8902","pr_number":6807,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-25T20:36:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tqsp5m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016712e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tqsp5m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016712e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4thdf.apps.aro.l4o4l3o0x3x9d8p.z2rt.j3793408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105598359803793408","occurred_at":"2026-10-01T09:59:03Z","signature_id":"ee987580b5aa868245a7803fc3f5acb34611d5f44d8e0253f6ca39cc99c1ec0b","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[2,0,0,1,1,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-25T16:19:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017f8e10\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017f8e10\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 3 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105804050531356672","occurred_at":"2026-10-01T23:36:24Z","signature_id":"af4881d75a9ae8226d8afbd4d63a88cbff508fb4ab0be92e201140cbc8343e5f","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T16:19:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:130]: HCP cluster rg-candidate-4-20-vgwv7thnmvhb/cluster-candidate-4-20-mvn65b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001036300\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-mvn65b: failed waiting for cluster=\"cluster-candidate-4-20-mvn65b\" in resourcegroup=\"rg-candidate-4-20-vgwv7thnmvhb\" to finish creating: GET https://rp.j8032640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8df1e89-0fd9-4b56-a005-77ef918b3bfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8df1e89-0fd9-4b56-a005-77ef918b3bfb\",\n      \"name\": \"c8df1e89-0fd9-4b56-a005-77ef918b3bfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:41:46.834189313Z\",\n      \"endTime\": \"2026-10-02T00:10:52.697527079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.40 is Partial (want Completed), started 1m39s ago; version 4.20.40 is Partial (want Completed), started 4m49s ago; version 4.20.40 is Partial (want Completed), started 10m52s ago; version 4.20.40 is Partial (want Completed), started 14m44s ago; version 4.20.40 is Partial (want Completed), started 19m16s ago; version 4.20.40 is Partial (want Completed), started 21m37s ago; version 4.20.40 is Partial (want Completed), started 23m41s ago; version 4.20.40 is Partial (want Completed), started 24m16s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:130]: HCP cluster rg-candidate-4-20-vgwv7thnmvhb/cluster-candidate-4-20-mvn65b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001036300\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-mvn65b: failed waiting for cluster=\"cluster-candidate-4-20-mvn65b\" in resourcegroup=\"rg-candidate-4-20-vgwv7thnmvhb\" to finish creating: GET https://rp.j8032640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8df1e89-0fd9-4b56-a005-77ef918b3bfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8df1e89-0fd9-4b56-a005-77ef918b3bfb\",\n      \"name\": \"c8df1e89-0fd9-4b56-a005-77ef918b3bfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:41:46.834189313Z\",\n      \"endTime\": \"2026-10-02T00:10:52.697527079Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.40 is Partial (want Completed), started 1m39s ago; version 4.20.40 is Partial (want Completed), started 4m49s ago; version 4.20.40 is Partial (want Completed), started 10m52s ago; version 4.20.40 is Partial (want Completed), started 14m44s ago; version 4.20.40 is Partial (want Completed), started 19m16s ago; version 4.20.40 is Partial (want Completed), started 21m37s ago; version 4.20.40 is Partial (want Completed), started 23m41s ago; version 4.20.40 is Partial (want Completed), started 24m16s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105791127218032640","occurred_at":"2026-10-01T22:45:03Z","signature_id":"b93d3d67354c2268d19256ee382276ea01db66fea349f1b5c476f8d2b3d14967","pr_number":7293,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b75130e68ab940da542a11fdc0fca87194c87f1940e2939766f8db2d841702ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The requested operation cannot be performed because the database account arohcpci01-rp-j2503040 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\ \\\\\\\\\\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0\"\\\\\\ \\\\ }; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-10-01T19:47:52.399Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-01T19:47:54.301Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-01T19:47:55.661Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=81d379c4f7c29ce92f1cd109cb00ebbe0457eec7d8a38ac495b54ae648addb81 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2503040%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F81d379c4f7c29ce92f1cd109cb00ebbe0457eec7d8a38ac495b54ae648addb81\ntime=2026-10-01T19:49:26.400Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2503040/providers/Microsoft.Resources/deployments/81d379c4f7c29ce92f1cd109cb00ebbe0457eec7d8a38ac495b54ae648addb81/operationStatuses/08584107228106955756\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2503040/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2503040/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j2503040\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"BadRequest\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The requested operation cannot be performed because the database account arohcpci01-rp-j2503040 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\\\r\\\\\\\\\\\\\\\\nActivityId: deddb421-d755-4f62-b0ea-94301557d50e, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105739854892503040","occurred_at":"2026-10-01T19:21:18Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6775,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,3],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T11:48:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:91]: failed to deploy customer infrastructure\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d79c0\u003e: \n    failed waiting for deployment \"customer-infra-delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-wx4qr5bhxpxx\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delayed-rbac-wx4qr5bhxpxx/providers/Microsoft.Resources/deployments/customer-infra-delayed-rbac-cluster/operationStatuses/08584107303732954713\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-usudlv6oegmxo' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:91]: failed to deploy customer infrastructure\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d79c0\u003e: \n    failed waiting for deployment \"customer-infra-delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-wx4qr5bhxpxx\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delayed-rbac-wx4qr5bhxpxx/providers/Microsoft.Resources/deployments/customer-infra-delayed-rbac-cluster/operationStatuses/08584107303732954713\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-usudlv6oegmxo' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105699867165724672","occurred_at":"2026-10-01T16:42:24Z","signature_id":"b99aaa56d60ccfaa0c0f746b012146616d725308307a216befe1c232cd129bb4","pr_number":6775,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-25T14:02:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002258720\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002258720\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n7r7f.apps.aro.cilium-cl.ni9g.j7451520.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105808059497451520","occurred_at":"2026-10-01T23:52:16Z","signature_id":"165618ea2ed8eed5bc91aac7d2d503d0fd826b5553de020ad73840a08512c1cc","pr_number":7314,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-24T06:52:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T07:22:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j1943424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1943424/providers/microsoft.monitor/accounts/services-j1943424\", namespace=\"arobit\", pod=\"arobit-forwarder-b8lv8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fe636689-1a0b-4733-829d-e26d9c7f983a\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-b8lv8 on cluster ci01-j1943424-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-01T07:22:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j1943424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1943424/providers/microsoft.monitor/accounts/services-j1943424\", namespace=\"arobit\", pod=\"arobit-forwarder-b8lv8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fe636689-1a0b-4733-829d-e26d9c7f983a\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-b8lv8 on cluster ci01-j1943424-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105536022291943424","occurred_at":"2026-10-01T05:51:21Z","signature_id":"9954a540c99e6058ff3c5a75998aed63f210d4dc026b87d4bc21149f2ba04252","pr_number":7283,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,0,2,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-27T12:48:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00262f770\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00262f770\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105693768928727040","occurred_at":"2026-10-01T16:18:10Z","signature_id":"ca8d78ae16088a003af0ff0522d3d50d3754aada3a46656a06c9179ae4952d6d","pr_number":7227,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"76fa49a860ae80b1247be5250347d4cda56628348e0ac69ad7bc4109f5092047","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013aaac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013aaac8\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105612041917042688","occurred_at":"2026-10-01T10:53:25Z","signature_id":"f7e25943d89dbbdc4ebfe0b130930a5207254c6e3ac44e8705a2ac207c9e0de2","pr_number":7288,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-v5pj4qqth2gl\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":21,"prior_last_seen_at":"2026-09-27T19:23:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000739be0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-v5pj4qqth2gl\" to finish creating: GET https://rp.j3797248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/523b7cc8-b248-458d-8692-3a89ce945aca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/523b7cc8-b248-458d-8692-3a89ce945aca\",\n      \"name\": \"523b7cc8-b248-458d-8692-3a89ce945aca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T13:50:45.474668418Z\",\n      \"endTime\": \"2026-10-01T14:09:46.073738279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-v5pj4qqth2gl--managed/providers/Microsoft.Compute/virtualMachines/n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-v5pj4qqth2gl--managed/providers/Microsoft.Compute/virtualMachines/n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000739be0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-v5pj4qqth2gl\" to finish creating: GET https://rp.j3797248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/523b7cc8-b248-458d-8692-3a89ce945aca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/523b7cc8-b248-458d-8692-3a89ce945aca\",\n      \"name\": \"523b7cc8-b248-458d-8692-3a89ce945aca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T13:50:45.474668418Z\",\n      \"endTime\": \"2026-10-01T14:09:46.073738279Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-v5pj4qqth2gl--managed/providers/Microsoft.Compute/virtualMachines/n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-v5pj4qqth2gl--managed/providers/Microsoft.Compute/virtualMachines/n7b6g4l4l8u2c3w-np-deletable-nlxsl-g2gld to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105632583583797248","occurred_at":"2026-10-01T12:15:03Z","signature_id":"84a2e933f9fa3e2bf4c49035661937c0be853d17d572ba5e196fb033e64d99b4","pr_number":7266,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-21T14:23:45Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-01T20:26:41.918Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-01T20:26:42.226Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-10-01T20:26:42.226Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-01T20:26:42.418Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-01T20:26:42.591Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-01T20:26:42.591Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-01T20:26:42.660Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-01T20:31:45.711Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-01T20:31:45.712Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105746107995787264","occurred_at":"2026-10-01T19:46:09Z","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","pr_number":7181,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"906f915d8d899da67c98c6d4cec8d92d5bbb542e88575bcce341f1bdb0d94049","failure_pattern":"node pool upgrade verification failed: \u003cnode\u003e (KubeletVersion v1","search_query":"node pool upgrade verification failed: h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-bdpxb (KubeletVersion v1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1627906976744187,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:205]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0014b5300\u003e: \n    node pool upgrade verification failed: h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-bdpxb (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-d2h5m (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-n4ktg (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); nodes=[{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-bdpxb\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":30},{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-d2h5m\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":47},{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-n4ktg\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":47}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:205]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0014b5300\u003e: \n    node pool upgrade verification failed: h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-bdpxb (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-d2h5m (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-n4ktg (KubeletVersion v1.33.13 has minor 33, expected 34 for OCP 4.21.35); nodes=[{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-bdpxb\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":30},{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-d2h5m\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":47},{\"name\":\"h6i8s8g8h0d0f1g-npupgrade-4-20-fq7kg-n4ktg\",\"ready\":true,\"kubeletVersion\":\"v1.33.13\",\"containerRuntimeVersion\":\"cri-o://1.33.13-7.rhaos4.20.gitd6a41c4.el9\",\"imageCount\":47}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105472600422486016","occurred_at":"2026-10-01T01:39:20Z","signature_id":"e45e129688a8baacf918d9fb33dc9b6e9fdccf4beea3a8e04b9c607f08d43cb8","pr_number":7260,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":34,"matched_occurrences":34,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":30,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":30,"after_last_push_count":40,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,10,30],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"09ad1536393e5c2e93d734e6e4a7baabe569c00b7a346b81e194a1dc526d6c84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"1e70068ca83e3da2caa181c14b566633dee772461c6bbef7e3b3c3b11c58ede2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"53d5243f6499b6bbb1f41dd260ea791225c721f0e2219068f0049f16fa9044cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"881e0d266cdbc141cde714bffad14ec487d883af4eafed8f23bce4c4912d543e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"8928cf54b630d0e6361d33389106d1e4b00e5b4fdda04424c7b1b3c9b344e25a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"93276e8c2e659949c04804f296318e19a7261db922d78b2b7a491384b13155c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"af5e31a50e667e6789ed1a04349bf851557b9188ce2f9b1e9df3b6b675fccd7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"e47528e4415d38ba10698d98e1ae80c6cfa1a4632c798890c9c25df89d4ae02f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"ee5d11d2818efb7c3c4ed23a4e01d5e6ab980563ce83dc358792a3f97e2d1f1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"fae23cdbef205ba19e74bef0e349a708114fea21f0bfce452cbd47ff26764f2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"3a4c53c8b8ed0f4e3a8c8625b638982616ccea702a79f9698e9303da79a15f67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"3f239f77699bb84da05b2be013b74a288bb4538f8ac6f15ec4ea749a60735ad4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"4b7a489ef3ea65128a354e82da4f4e1de6bc22ca63444681abe780ccc7ddf4d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"9ed87e9e9315982414d4d556eba0d0bea3967a50c31ea93eff6a36c532a8eb2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"a23758f9b4de17fd810383f0e57b42b4438075c36125aa9b6c712edcb5ea75a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"a5ad9b8fa8feb1257e28216d87d1a08d738224a79e57266bf4690f58f661ad42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"ad3a6f95d691523f560574aa964dea297931db1f1651191d1fcc28b85649a55a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"b42e83d1c1e188b5a53a98054e2e876e0d0416f2e4658e9a37a24c36e12854fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"f393133e89a119909f1414ed37aedeb249de63da9461ad1d5feb1022be0d1b09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"ffa6512d457f2bdedb8df8b3e86503ab5c120285bf88c97a42b1b851b87335e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"0ec303987eddb2fdc7951010bee64e64f19b2795c36b27fa9b6c16d511039f6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"10b16b7588fc77933a91ee23f48c974f59b3cf6c7796512f1986ba476c2fdcba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"535d9b5810530eb8fa93d005fe5342bc6c5f76b1ac4619d0de9e457c08ad121a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"652af8ac6d7caa46ada925ea05e4b4f678aa2fa90a5c6290737d7b660b7f2248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"a2c9210be7e28a9733e941546fc75b632c067fbedccb9d0a6c5fc389f9a2e48f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"a8535460c6d3cd45df9df1883152d24842155b835833878a66f7b2d3ba6aaa58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"bc6beae7f23eaeb00f646e9afe501ded4abd7941cafe250a38a63603361caf1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"bc99803c563848f8ef2751f1a43b94eaeabe40a9b89129e706b11fea21a2a78d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"de5390b7de0f9fb6b5b999f0840b119c4cfcea4837064e22c1a37c81aa8301d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"eda6d7fa1b7ee822da5e9598ea4cc8fe8832dbc41c8d88d803855760b7f64988","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":11,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,1,1,4,2,3],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-27T07:04:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n      \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n      \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000941160\u003e: \n    failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n      \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000941160\u003e: \n    failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n      \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n      \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n      \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","signature_id":"24417d5ef077cde226947335837e6d950a92fb685e28c57995b46f4508d8e565","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","signature_id":"a7f9c67af898546f71a808604e7639e1ebc1c0ed5391f8286fb71f6bda1d5873","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"e33b49f25d1ec8838fb3fb2248feed144429268d006904054aadad3f41b2c4ef","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-26T21:16:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n      \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n      \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"44b6707750ec56079daaa0398aa289b9339df5e0faca1b38d9f04663d954b374","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":12,"failed_runs":6,"raw_occurrences":33,"matched_occurrences":33,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"5de8ba5f528a28280d389eef3564abae9054b9a2afb4d837caec5c56528c05af","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["INT","STG"],"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4mwqz2\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jlsd79zkk6hh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4mwqz2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4mwqz2\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jlsd79zkk6hh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4mwqz2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-4fwf7q\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001226630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wnp2jmpdpwln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4fwf7q\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-4fwf7q\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001226630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wnp2jmpdpwln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4fwf7q\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-24sw5t\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cdd5c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vhz5sh8l55g9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-24sw5t\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-24sw5t\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cdd5c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vhz5sh8l55g9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-24sw5t\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"19e49d7275216187415596a77e7c4bcf8f328fed8c07e3decc611480955564e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"412c1d488719dd5fc967130e3fa6effa7e82d9edd5a17a5e8176ab4d70765e0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"4d9b0ad4d995439a69761daa6502428708fa7e25caed34c232065582abd6a8be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"5ad91d5ba592c718110d060ec85bbce1cf9bc9da9a852c065bb122315fa608f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"7d8e65006d2598e018d984d2201d99a02258f21049c17c9d62e58cf6c40e840a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a3f652fc0d3cf37233e1a99bcbf33f40c66eb18489b226915716ce4457c123d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"ca31150f8c635319cf607500a541ad8b66476dc3c1106a8a6a766f27645ff9cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"d071cb6535a5544c2ea6eef944d6e2c7978595749be12c29f68a5b0ef11e7d20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"dcc2d2f958cc4b5a06713c7dd534ea32a664855ca4cc18435215b00e2e387fe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"df47e1d06972d594dea943756cc7419be51fa395cd53a06b8f50e809684c14b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":22,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,4,10,2,6],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001797968\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7fp2gpvrmpv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"name\": \"fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:17:05.438751852Z\",\n      \"endTime\": \"2026-10-01T23:36:05.105086237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001797968\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7fp2gpvrmpv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"name\": \"fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:17:05.438751852Z\",\n      \"endTime\": \"2026-10-01T23:36:05.105086237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbe8e0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-f85th5\" in resourcegroup=\"rg-np-noedge-f85th5-2lj4rn8xdzrh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"name\": \"3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:02:34.984361176Z\",\n      \"endTime\": \"2026-10-01T18:21:38.6870989Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbe8e0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-f85th5\" in resourcegroup=\"rg-np-noedge-f85th5-2lj4rn8xdzrh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"name\": \"3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:02:34.984361176Z\",\n      \"endTime\": \"2026-10-01T18:21:38.6870989Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010769c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g5k9xrk5kbzf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"name\": \"1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T16:25:59.009326346Z\",\n      \"endTime\": \"2026-10-01T16:45:03.114189778Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010769c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g5k9xrk5kbzf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"name\": \"1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T16:25:59.009326346Z\",\n      \"endTime\": \"2026-10-01T16:45:03.114189778Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"5a88e8e35a16faf5ee7a4d5b4f6f32a8c37dfc00d34795932e2e514c6e30e085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"27caddf4f1e8f34eee4fc2b6ae6d34e392067680761ae86d5ecb6266597fed04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"61d0c9edd234b7e14042d3032095db877dbe6323fb17ce8fad8a175cc612fec3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105520036851486720","occurred_at":"2026-10-01T04:47:40Z","signature_id":"814a16bef1beba6789dffa5698353084a9348129da2a78e7729f185a8f59dec9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"ad9369c416904d0bc5cff3fb32e4116dd601fade18d109c2a14774ed4b1186d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"ed74568737500cd622a695c5ad75e81af359673885d49c87f038f5674080cfca","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,1,0,0,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-47sckvjgzdjp/cluster-nightly-4-22-wwdkq6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-wwdkq6: failed waiting for cluster=\"cluster-nightly-4-22-wwdkq6\" in resourcegroup=\"rg-nightly-4-22-47sckvjgzdjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"name\": \"4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:33:29.768708196Z\",\n      \"endTime\": \"2026-10-02T00:02:35.156751805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-47sckvjgzdjp/cluster-nightly-4-22-wwdkq6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-wwdkq6: failed waiting for cluster=\"cluster-nightly-4-22-wwdkq6\" in resourcegroup=\"rg-nightly-4-22-47sckvjgzdjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"name\": \"4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:33:29.768708196Z\",\n      \"endTime\": \"2026-10-02T00:02:35.156751805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-szptjh\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f34e40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-szptjh: failed waiting for cluster=\"cp-ystream-upgrade-4-23-szptjh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-46pm69d84hvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"name\": \"780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:37:58.369447145Z\",\n      \"endTime\": \"2026-10-02T00:06:59.154886423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-szptjh\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f34e40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-szptjh: failed waiting for cluster=\"cp-ystream-upgrade-4-23-szptjh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-46pm69d84hvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"name\": \"780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:37:58.369447145Z\",\n      \"endTime\": \"2026-10-02T00:06:59.154886423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-bjmvhq at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010869e0\u003e: \n    failed to create HCP cluster np-skip-minor-bjmvhq: failed waiting for cluster=\"np-skip-minor-bjmvhq\" in resourcegroup=\"rg-np-skip-minor-bjmvhq-v7kckbmqz7jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"name\": \"94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:05:39.688922023Z\",\n      \"endTime\": \"2026-10-02T00:34:39.296960872Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-bjmvhq at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010869e0\u003e: \n    failed to create HCP cluster np-skip-minor-bjmvhq: failed waiting for cluster=\"np-skip-minor-bjmvhq\" in resourcegroup=\"rg-np-skip-minor-bjmvhq-v7kckbmqz7jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"name\": \"94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:05:39.688922023Z\",\n      \"endTime\": \"2026-10-02T00:34:39.296960872Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"284badc75c52f2c928896b544484ccaf81523055e480cbb01f205d32fbfb2055","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"6906776b10e91fb1930393db0f67c0e5c85ddc0131f15dc7f553da0111c50b64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"6db1e852eea8dba2acd65aead4a02f26184108da6a19027e7eac3481346d7608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9693157aa0c8e52231f85346439be27795bb96ffd3506957078219d7289eee55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"a47958a0b27a7e6417393acfe801cf47768b5d3548c940923347b671dfb151db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":5,"after_last_push_count":68,"trend_counts":[5,5,0,5,5,0,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f4b7a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-5rjr485pnvns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f4b7a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-5rjr485pnvns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013b7a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-55xcrpmcpjd5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013b7a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-55xcrpmcpjd5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a6f140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwrdblk56ldw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a6f140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwrdblk56ldw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"0879af2742cd30b1af5adb170d9ba043c2ac72dca9e6206fd1ad2ce6a2f70845","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"167b28b3e3cf18e7829a5045c71ef1603c48da5ee1faa9e604ee810dca39cf48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"1b8a88d9699db38c5538a15c857505c8cfc1b3d077c382a6629566c882287c82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9743f5915e7ad3e053054c21c8eb514f30668f1dd7765974f91761e8bad2e327","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9bd45a89f4ba89d2fe993c417db2bf0c1d7eb96c4b27a4fc8ab08b4827e95748","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,2,1,0,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f80160\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f80160\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012cb640\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012cb640\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"721a2923a68f98f8036e86221e342512b007d654edab41bc4b01d104c7c31069","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"42071066d578584734f0ff88f8d91844d63d17cae6fd1ba95e338cf5b297b9ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":2,"after_last_push_count":56,"trend_counts":[4,2,4,4,4,4,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed4500\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ngvxnx\" in resourcegroup=\"rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ngvxnx/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed4500\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ngvxnx\" in resourcegroup=\"rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ngvxnx/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012f2520\u003e: \n    failed to create NodePool np-ks75xq: failed starting nodepool creation \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\" in resourcegroup=\"rg-nightly-4-20-2zfvw4gzpm77\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2zfvw4gzpm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-ks75xq/nodePools/np-ks75xq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012f2520\u003e: \n    failed to create NodePool np-ks75xq: failed starting nodepool creation \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\" in resourcegroup=\"rg-nightly-4-20-2zfvw4gzpm77\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2zfvw4gzpm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-ks75xq/nodePools/np-ks75xq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"433fa8bcb3330f342d959be31de105f077c4ee1f9cf1595e004254245cb6d202","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"fb02c473ebd0d6e66f0313e3e4cc1a3da903edced25d6fe7248861c17e69533c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,0,4,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000638600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nzmgkxwzltg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"name\": \"eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T05:17:35.937388036Z\",\n      \"endTime\": \"2026-10-01T05:36:39.648087048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000638600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nzmgkxwzltg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"name\": \"eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T05:17:35.937388036Z\",\n      \"endTime\": \"2026-10-01T05:36:39.648087048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105520443560562688","occurred_at":"2026-10-01T04:49:27Z","signature_id":"42138565829011884507a259879669579b12c563a2f4a280f0c5c88587c8f9f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001802258\u003e: \n    resource group \"rg-zstream-upgrade-4-20-26f5nw6rjl84\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-m4xqll\" in resourcegroup=\"rg-zstream-upgrade-4-20-26f5nw6rjl84\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"name\": \"fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T17:28:43.347525555Z\",\n      \"endTime\": \"2026-10-01T17:52:46.138429989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t648pvv6ett9l7cgmh574bcs7i8oe4q still exists (deletion dispatched at 2026-10-01T17:28:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001802258\u003e: \n    resource group \"rg-zstream-upgrade-4-20-26f5nw6rjl84\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-m4xqll\" in resourcegroup=\"rg-zstream-upgrade-4-20-26f5nw6rjl84\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"name\": \"fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T17:28:43.347525555Z\",\n      \"endTime\": \"2026-10-01T17:52:46.138429989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t648pvv6ett9l7cgmh574bcs7i8oe4q still exists (deletion dispatched at 2026-10-01T17:28:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"9cd553a21a7a83d96159492dd3d1993283938905adb80a47e77311686d9f1acf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":21,"trend_counts":[1,2,2,1,3,2,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014996b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-10-01-083859, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014996b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-10-01-083859, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"d6fefeb9b4d855c8f485e15a2e9d74970a7f65266f90d3948da1d4fc0e6c9484","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":28,"matched_occurrences":27,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["INT","PROD"],"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00164d620\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00164d620\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"11a7f777efd16a4602f165155402df79554a160ce761b48987b2fa81eaa853ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"1719e4976d13747a5ba884e6917c1488f67a8a955b14f27daff4e13ed2902037","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"2388c2f6af983771920940504bcf433d725fb8c7cc1e81c55f6ab467636ed7ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"5565e5f16c645b130d162f8ebf645832f59d83ad60615990833bdafaca4b6ef8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"844c1e8cf0cc01d92e68b7c7011864371b606ef8f2d55622248024c05d7ce83e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a09d9f975ede859b703696d2a88baac07d837fd9c31b0ebe46690181381da93a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a8eb2d795a11c6dd632258debe4cb2cff59cae61bb73a703f5d0ea485effd6ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"c14433cba7a1a57c209cdd5665919f16d75f3d85ba2502e0034b92b8f4aee701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"f03158936394395704c5b6ff64b4370dd9e2548b869de7e69671c413ff75621f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"fd3be8a2cabe6e92eed1d71ec31a986980accf6d34064931d1b31fd3558ce497","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":8,"after_last_push_count":8,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n      \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n      \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f200c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n      \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f200c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n      \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012e40\u003e: \n    failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n      \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012e40\u003e: \n    failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n      \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"1b39a1fcea6ce322dd8fa1bd054343dbd452f0f0491cacc4f8b9ef7b6a0493be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"2169238a3efc65d2f002e87d4107ee839610a68181a3b21a5f5f8555c8c0152b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"261d2ba977979da2043d1d05e7fda09b0050628d435b4e8f6c70c65d7ede0f72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"4247285a4a7e0ce74bb11c7155b281ec95aa3b9b5e58b9fb3220d4d61dc786b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"577a2f2cdd3de8213f2886c7670e9678f9d789b1a765ea47d5053e8926ce155f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"84ad30e1bfed5ae9658a5da88f0ca1af31ae55f3c693883c462c46dcf7608e01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"b4159087ef39a5abb0c42cbb58f2ea45413e0c9d5e74cdc7078f52b05a7b45c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"bdfd444f6b247ef48624113640b671ccd50695d691f9d049b710ad7273122875","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":6,"after_last_push_count":37,"also_in":["DEV","INT","PROD"],"trend_counts":[0,17,1,2,1,10,6],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n      \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n      \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a16440\u003e: \n    failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n      \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a16440\u003e: \n    failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n      \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f300\u003e: \n    failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n      \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f300\u003e: \n    failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n      \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"153a3254e17592610cd68e651bfb6cbf2394189d89044802b8c3b533e149203a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"6fc83d6684638cb567013759a7193f362c34949e41b3a8668c62f64a251d057c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"623dfd830d068e8fc297a04ee4a56d7cb6e4821c0251af7874f2b77ba7c26fa6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"c574b07b3114ca80e839d7f5d60517775ee576de2d943538a90489d57cb3eec5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"f47c5337335a014c2bed2f388006a403bbb00ccd35a477f73d5472519ff75085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105450319994425344","occurred_at":"2026-10-01T00:10:48Z","signature_id":"b47c99754a6a1f045aa9bf003dd005e8d53c7b6f39de576acac3bb76b9fadb06","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c16198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n      \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c16198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n      \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"951b0a5c5a70e3b35bb3f64267eb4dc0e2b6817b2864d1a40fa29654b0ff2a81","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019901c8\u003e: \n    resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n      \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019901c8\u003e: \n    resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n      \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"307bc38fc0da1536f942492cf74c8e559878301b0fc5629d37255c20e9ee7ec4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000d82400\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n      \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000d82400\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n      \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"bff4c5f332cfa283a8daa21713459950af8584404733577995b07a2a2bfcc5a8","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}