{"meta":{"start_date":"2026-10-02","end_date":"2026-10-02","start_at":"2026-10-02T00:00:00Z","end_at":"2026-10-03T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:14:42Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":66,"failed_runs":48,"raw_occurrences":96,"matched_occurrences":94,"runs_affected":46},"rows":[{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":21,"runs_affected":19,"failed_runs":19,"run_impact_percent":28.78787878787879,"anchor_occurrences":21,"after_last_push_count":32,"trend_counts":[2,7,19,10,8,21,21],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":82,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001182600\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt\" to finish creating: GET https://rp.j6364800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"name\": \"6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:01:24.493949107Z\",\n \"endTime\": \"2026-10-03T00:20:29.737570785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001182600\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt\" to finish creating: GET https://rp.j6364800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"name\": \"6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:01:24.493949107Z\",\n \"endTime\": \"2026-10-03T00:20:29.737570785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad2a60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9h4twv\" in resourcegroup=\"rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx\" to finish creating: GET https://rp.j9301120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\",\n \"name\": \"798eb1c5-8054-497c-b466-6bdb523be072\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:16:07.311984652Z\",\n \"endTime\": \"2026-10-02T23:35:13.399526187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad2a60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9h4twv\" in resourcegroup=\"rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx\" to finish creating: GET https://rp.j9301120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\",\n \"name\": \"798eb1c5-8054-497c-b466-6bdb523be072\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:16:07.311984652Z\",\n \"endTime\": \"2026-10-02T23:35:13.399526187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d1b20\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-b86qkn\" in resourcegroup=\"rg-np-version-upgrade-b86qkn-wdjflpkjvp8p\" to finish creating: GET https://rp.j0034432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"name\": \"ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:59:34.896590709Z\",\n \"endTime\": \"2026-10-02T22:18:38.199181809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d1b20\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-b86qkn\" in resourcegroup=\"rg-np-version-upgrade-b86qkn-wdjflpkjvp8p\" to finish creating: GET https://rp.j0034432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"name\": \"ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:59:34.896590709Z\",\n \"endTime\": \"2026-10-02T22:18:38.199181809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157602076364800","occurred_at":"2026-10-02T23:01:13Z","signature_id":"34de2ab11f5adbf068c342e2d31a9ac77891b0e38b007bfdbb6d9da4bc98f473","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141275559301120","occurred_at":"2026-10-02T21:56:24Z","signature_id":"63d927cd1c7a9670c7278fe12e140bf43f44a0266f7ac91ced09dcf4f09830bc","pr_number":7278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","signature_id":"ce7070d24262cdcf05d41676703c04f5be7f690258e7e4c7e074539c5acf5075","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7327/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106114529938116608","occurred_at":"2026-10-02T20:10:08Z","signature_id":"5b5e8add1f118eb1fc026e437f941636e24ad6dd0bfaf6584e1d203e456a64d8","pr_number":7327,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109131696902144","occurred_at":"2026-10-02T19:48:41Z","signature_id":"f88616dce735dbe299ec31741c0e4e4dd95b26a9ca444ca47280bbcc48727d97","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","signature_id":"1983f452786e60abbdb725b8622697151a96361ebb27e00c6a2b0a244b8f9d74","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081336937680896","occurred_at":"2026-10-02T17:58:14Z","signature_id":"31dc382fb43fe26f7ce7e41de756fcc58f0efe54d9f97db17cfd4bb76fb55a77","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","signature_id":"80687d33a4d52dba803f3167e5984e21a97d039dd3cf4af22716d553b7595b37","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","signature_id":"6c5905a125f03a9ccb5e9ff9fb30a16eb39c975e893c7c332e674d06be66eefe","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035428787228672","occurred_at":"2026-10-02T14:55:49Z","signature_id":"bc5be3ac53788be83982476868c7fc5ebab96a3431e2d77151fdef86774b0791","pr_number":7314,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","signature_id":"e55a99665469ae5bdd51816697d8d92b366d93c347a1df665aa039f3c7dfc45f","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106027308732125184","occurred_at":"2026-10-02T14:23:33Z","signature_id":"a03b4f13d1b794c0ce5f7e27910be2a2b3cf8e2efc010caf5845fbed77edcc07","pr_number":7323,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989665596116992","occurred_at":"2026-10-02T11:53:58Z","signature_id":"aca469fe140764c02ab7856c4660d7eff0c7a2566b4afda4da099d345dece618","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","signature_id":"1c44ed3ef9cc244959890b608e75fe48aace2cea67f57efb73c27a002b3dc074","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","signature_id":"30cfc0a39a792902039ad9670c2f071e0a56ecab7b20da519e046ac01486e359","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"3a9b1d0a1756d4352283ba3b63e3d886955fa9c5c336253e1b019206952a5507","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"c4fa5c26ee982b83676c3b7397954d31e709a456e87dee9158b60d5d5dfcd62f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105899086866550784","occurred_at":"2026-10-02T05:54:02Z","signature_id":"452eed4657b73553008ac22f54a6f1e9229b5abe0497e58b51b804e7a82c75ad","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860632891887616","occurred_at":"2026-10-02T03:21:14Z","signature_id":"79f30e35241b4c02e2ba6c027a86d8d186f0dd2acc5e0e41d5fd756e33a11c74","pr_number":7314,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860557423775744","occurred_at":"2026-10-02T03:20:56Z","signature_id":"c762968dfb2da1a7708f604dd85763b8611979abccdf3d8a74f487a345b560c1","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105811761943810048","occurred_at":"2026-10-02T00:07:02Z","signature_id":"e68aa4f1fd5584a77fb4d1bf176cf9e57b599009a951da1f3d48eac028964cfb","pr_number":7105,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":19,"runs_affected":19,"failed_runs":19,"run_impact_percent":28.78787878787879,"anchor_occurrences":19,"after_last_push_count":32,"trend_counts":[0,2,7,11,3,5,19],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":84,"prior_last_seen_at":"2026-09-27T13:27:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":19}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:06:25Z\n Ended: 2026-10-03T00:13:29Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6364800-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:07:24Z\n Ended: 2026-10-03T00:19:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7564928-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T00:08:27Z\n Ended: 2026-10-03T00:27:27Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7564928-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T21:34:19Z\n Ended: 2026-10-02T21:40:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8116608-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157602076364800","occurred_at":"2026-10-02T23:01:13Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157368797564928","occurred_at":"2026-10-02T23:00:21Z","signature_id":"04501c7bb397428ef39e831a7c967076ecd521dff084b9e5def70ae9b291e02a","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7327/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106114529938116608","occurred_at":"2026-10-02T20:10:08Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7327,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109176676618240","occurred_at":"2026-10-02T19:48:51Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081367363162112","occurred_at":"2026-10-02T17:58:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081336937680896","occurred_at":"2026-10-02T17:58:14Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106078475705126912","occurred_at":"2026-10-02T17:46:52Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7323,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106064405874610176","occurred_at":"2026-10-02T16:50:57Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106050489337516032","occurred_at":"2026-10-02T15:55:39Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7321,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035454687055872","occurred_at":"2026-10-02T14:55:55Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106027308732125184","occurred_at":"2026-10-02T14:23:33Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7323,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106024077666816000","occurred_at":"2026-10-02T14:10:42Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106020738229080064","occurred_at":"2026-10-02T13:57:26Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106003628014702592","occurred_at":"2026-10-02T12:49:27Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7309,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105933227649667072","occurred_at":"2026-10-02T08:09:42Z","signature_id":"1f3b34a45cf8a5ee02900fdfe7d8f46291016a5dd0e2aac524d7b4e070d18d47","pr_number":7293,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105916967100616704","occurred_at":"2026-10-02T07:05:05Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fskm94\" on minor version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32b40\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-fskm94\" in resourcegroup=\"rg-zstream-upgrade-4-20-7d7sxcc5nbp4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"name\": \"0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:38.87341489Z\",\n \"endTime\": \"2026-10-02T23:43:46.612931387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fskm94\" on minor version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32b40\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-fskm94\" in resourcegroup=\"rg-zstream-upgrade-4-20-7d7sxcc5nbp4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"name\": \"0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:38.87341489Z\",\n \"endTime\": \"2026-10-02T23:43:46.612931387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-59v6sn\" on minor version 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be1060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-23-59v6sn\" in resourcegroup=\"rg-zstream-upgrade-4-23-dwdrgs9rpkt4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"name\": \"f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:00:29.412121269Z\",\n \"endTime\": \"2026-10-02T23:44:36.589759019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-59v6sn\" on minor version 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be1060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-23-59v6sn\" in resourcegroup=\"rg-zstream-upgrade-4-23-dwdrgs9rpkt4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"name\": \"f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:00:29.412121269Z\",\n \"endTime\": \"2026-10-02T23:44:36.589759019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6f800\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xzgkv9b984lz\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"name\": \"bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:51.980435313Z\",\n \"endTime\": \"2026-10-02T23:08:01.293538008Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6f800\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xzgkv9b984lz\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"name\": \"bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:51.980435313Z\",\n \"endTime\": \"2026-10-02T23:08:01.293538008Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"0660e5aeb2a877fab962d22ed49cccd7985f44881bfda87726aaa352b96da22a","pr_number":7313,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"4d059280747339409745ae35874bcfc7c6f8797f43ab108053ff4751fb5cfa9b","pr_number":7313,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"5bd9a565d39a5a143cbf016e6eacabe5fc41e6b584fd89893528ebc43bc9173f","pr_number":7313,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"affbacfd51e93098bd3964ec613491d9c1e15b85104d19de75d8c835e704fdce","pr_number":7313,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"b7678a0e4b4e24474229f76609c785b6fb93d2d584801f6c25d6ac0623415d59","pr_number":7313,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","signature_id":"ca2142bbb3700266ca07c7e4c24330f499e20b7321800d8b3c755ba6c4b16e28","pr_number":7313,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":7.575757575757576,"anchor_occurrences":5,"after_last_push_count":64,"trend_counts":[1,2,1,7,10,6,5],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":132,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":5}],"full_error_samples":["alert fired 9 time(s) (1 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:22:54Z\n Ended: 2026-10-03T00:28:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7564928-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6vba5aukt21s79ka316m1hqn628hho-s4a5z1v1k7m8i6h\", pod=\"openshift-apiserver-64ddcdd5c4-tjv4j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6vba5aukt21s79ka316m1hqn628hho-s4a5z1v1k7m8i6h/openshift-apiserver-64ddcdd5c4-tjv4j has been in a non-ready state for longer than 5 minutes.","alert fired 40 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T20:04:20Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-rqmvp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-rqmvp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T20:04:20Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-pclwm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-pclwm has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-ffmdk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-ffmdk has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-lqbwl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-lqbwl has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7xqdb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7xqdb has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-xfn2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-xfn2j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-hrb67\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-hrb67 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-9rzfn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-9rzfn has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-8rlch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-8rlch has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-xz6lm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-xz6lm has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-psr9c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-psr9c has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-tdg92\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-tdg92 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nnzlj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nnzlj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-cp4cg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-cp4cg has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-zs2jk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-zs2jk has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-bft9l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-bft9l has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-l6fg8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-l6fg8 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nkfbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nkfbd has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-zqdt7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-zqdt7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-6hf78\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-6hf78 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-v2424\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-v2424 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7xlbs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7xlbs has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-gr2nd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-gr2nd has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-hhkrx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-hhkrx has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-wnr5j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-wnr5j has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-6kxrp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-6kxrp has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-wxmwp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-wxmwp has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-dgvs2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-dgvs2 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nt5f5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nt5f5 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-qldfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-qldfs has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-vjmrq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-vjmrq has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-mz8lv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-mz8lv has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7r5rf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7r5rf has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-4ljs8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-4ljs8 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-vp8px\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-vp8px has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-t8hh6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-t8hh6 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-9xv2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-9xv2x has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-w4rxn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-w4rxn has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-mfxhq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-mfxhq has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-vngdx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-vngdx has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T17:21:19Z\n Ended: 2026-10-02T17:32:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6990720-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6pa7oq39hmo04tlm81048rosc9slps-ea-listt98lhd\", pod=\"kube-apiserver-594c7bf48-44xbg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6pa7oq39hmo04tlm81048rosc9slps-ea-listt98lhd/kube-apiserver-594c7bf48-44xbg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157368797564928","occurred_at":"2026-10-02T23:00:21Z","signature_id":"fa3c4dd358a0d346db704e5b27ea7256c0e149b36b853364eed04a48b1ad2821","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","signature_id":"7deb638e8e55b55194c14aa0cd42ee58586132dbbb6f2d7f6d32b6dbaa200027","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","signature_id":"9af0e1f2b1b242d263756ff45f69a3d0f7754608d5789a9bf850d6d14b180404","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","signature_id":"6c5480d9d7704b8a0e81ba15f14a4d9edb6d6b1ff0aff158900e2a7bd3221612","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"816db2d3fb1d50b6391b3d4ae23ae87f6b66058b291be9d7e3fc5c359fca5e80","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.0606060606060606,"anchor_occurrences":4,"after_last_push_count":3,"trend_counts":[0,0,2,0,1,0,4],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-25T20:17:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2640\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2640\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001e820c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ftx74z7plhzf\" to finish creating: GET https://rp.j1487616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"name\": \"737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:53:28.478249577Z\",\n \"endTime\": \"2026-10-02T20:12:35.176603143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001e820c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ftx74z7plhzf\" to finish creating: GET https://rp.j1487616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"name\": \"737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:53:28.478249577Z\",\n \"endTime\": \"2026-10-02T20:12:35.176603143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdc160\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdc160\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141275559301120","occurred_at":"2026-10-02T21:56:24Z","signature_id":"7060bb637cf275cb6e0005a64cdfdea3e58f04131055a1a620001107ea86bc5a","pr_number":7278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106096143061487616","occurred_at":"2026-10-02T18:57:04Z","signature_id":"75c54a19ff7c88fe534fc97b238d4fa34e6bdf28dcd643fa62b5d5f8c5dde05b","pr_number":7321,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","signature_id":"03ee416419f4f30380384e985dc45aaa36ef7f2e21c2accfb48e05ac0da1c153","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989639176196096","occurred_at":"2026-10-02T11:53:51Z","signature_id":"59a6c9ed7c614c1f50c5055404a7b13ac0ce911ce97dad36bf7ce17f21a3947f","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.545454545454546,"anchor_occurrences":3,"after_last_push_count":15,"trend_counts":[0,0,0,41,1,0,3],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":45,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001047368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001047368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011448e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011448e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000dc15d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000dc15d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","signature_id":"5c9873debb2342170eed4903e7f228938c3a34d04b4f651f99d6b862372dfffb","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","signature_id":"fd4d0de36426d6f1b5142554b466d73d39b32478872d976808a351b06917f2e4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"247f2f7e907043573a578d118d78f6c6f00500e34fc61faec7dbd664db892fcf","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.545454545454546,"anchor_occurrences":3,"after_last_push_count":8,"trend_counts":[2,1,5,6,7,0,3],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":42,"prior_last_seen_at":"2026-09-27T13:08:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3}],"full_error_samples":["alert fired 65 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-22dl9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-22dl9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dfssq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dfssq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-lc7f8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-lc7f8 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-lrxrn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-lrxrn has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ws459\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ws459 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-nm4b8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-nm4b8 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-r88sm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-r88sm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-2f87w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-2f87w has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-hlcf4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-hlcf4 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6mnrg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6mnrg has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-t95zp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-t95zp has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jkbjb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jkbjb has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zcpgg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zcpgg has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-qxvct\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-qxvct has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-mfbkm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-mfbkm has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-mmpwv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-mmpwv has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-m4v82\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-m4v82 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-fzf98\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-fzf98 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6f2b7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6f2b7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-z9cp6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-z9cp6 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-64mk2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-64mk2 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-glxpz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-glxpz has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-scxrj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-scxrj has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5876w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5876w has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-h5skh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-h5skh has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-77rvv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-77rvv has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-wxzgc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-wxzgc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gxvrp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gxvrp has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-f497d\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-f497d has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zpfft\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zpfft has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-x99lx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-x99lx has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ksr8r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ksr8r has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-p2k9c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-p2k9c has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-hp5fb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-hp5fb has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vjh72\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vjh72 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gfrgk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gfrgk has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-nrh6c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-nrh6c has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dcz7s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dcz7s has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-56n2t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-56n2t has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-8pwww\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-8pwww has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-sr95r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-sr95r has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-sl2gh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-sl2gh has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dlbhp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dlbhp has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-pbjdw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-pbjdw has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-rfjgp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-rfjgp has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-xwfwj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-xwfwj has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gl2xg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gl2xg has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-96lg5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-96lg5 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vbcxj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vbcxj has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5sw59\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5sw59 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-l49jm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-l49jm has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jx5v4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jx5v4 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-s256n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-s256n has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vnr48\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vnr48 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zrk5n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zrk5n has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-cc7rs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-cc7rs has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6j84r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6j84r has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-blmqs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-blmqs has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-4lnm2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-4lnm2 has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ppjmp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ppjmp has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-t9nwg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-t9nwg has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-68bhc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-68bhc has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5fhnt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5fhnt has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jjrn5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jjrn5 has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-rzbk6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-rzbk6 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T15:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9111424-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-k7gkp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-k7gkp has been in a non-ready state for longer than 5 minutes.","alert fired 5 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-wzq9v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-wzq9v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-qj4ln\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-qj4ln has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-w44fw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-w44fw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-66rf9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-66rf9 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-4p6tc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-4p6tc has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","signature_id":"8a0e5803002408b9740f03b409808525c6d5017ddf44ce6a4187ef72194ff101","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","signature_id":"c4307b9de315687a8fce104eff903477686b7321152ba52d83a069c63664370a","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860601363304448","occurred_at":"2026-10-02T03:21:06Z","signature_id":"292f546bc904e43ecc8422922e08247071ec4dff8f7132975575642549b9e4ef","pr_number":7105,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.545454545454546,"anchor_occurrences":3,"after_last_push_count":10,"trend_counts":[4,4,7,1,0,0,3],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":32,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106171709915664384","occurred_at":"2026-10-02T23:57:21Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa","failure_pattern":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.545454545454546,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.006s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-d97ndt\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001ec6360\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.006s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-d97ndt\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001ec6360\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.001s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-qjs7pm\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011ce348\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.001s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-qjs7pm\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011ce348\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.009s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-dg8wqw\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015c3050\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.009s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-dg8wqw\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015c3050\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","signature_id":"24d9756fc06393ff60a7f26c3d09a05911ada4575a0de58939f9e9ec06ce0777","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","signature_id":"0a5b2526c26b68cdf55e2a3874db601a881bf79208555f69ca0b1f64f834031b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"9b85a307c626519801ac209425f1debdf469e99c3d127b2e7e94b3fca1fbcb53","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.0303030303030303,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028843f0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kqrgzxwnxwgd\" to finish creating: GET https://rp.j8271104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"name\": \"7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T12:59:29.315762309Z\",\n \"endTime\": \"2026-10-02T13:18:36.006994146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-597b9-prdsc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-597b9-prdsc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028843f0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kqrgzxwnxwgd\" to finish creating: GET https://rp.j8271104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"name\": \"7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T12:59:29.315762309Z\",\n \"endTime\": \"2026-10-02T13:18:36.006994146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-597b9-prdsc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-597b9-prdsc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84f00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2vzgl45q9ll4\" to finish creating: GET https://rp.j0616704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"name\": \"992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:27:53.917230714Z\",\n \"endTime\": \"2026-10-02T08:46:59.056787427Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84f00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2vzgl45q9ll4\" to finish creating: GET https://rp.j0616704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"name\": \"992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:27:53.917230714Z\",\n \"endTime\": \"2026-10-02T08:46:59.056787427Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989632788271104","occurred_at":"2026-10-02T11:53:50Z","signature_id":"f69ba6188909b28f74884c51e52b94a32d60de724f0f21d0e969c7c8782b6c2f","pr_number":7314,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105916967100616704","occurred_at":"2026-10-02T07:05:05Z","signature_id":"aa94cf88bb79706f28a4d806b8b9860fe56b6477bb4b62bfc09ab3f7354f8d08","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.0303030303030303,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,0,1,2,2,3,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-21T22:11:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T22:50:00Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0034432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0034432/providers/microsoft.monitor/accounts/services-j0034432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-fdbccb774-c7nq9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0034432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T22:50:00Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0034432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0034432/providers/microsoft.monitor/accounts/services-j0034432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-fdbccb774-c7nq9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0034432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T19:11:26Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4610176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4610176/providers/microsoft.monitor/accounts/services-j4610176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-69f9b555fd-7q6cb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4610176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T19:11:26Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4610176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4610176/providers/microsoft.monitor/accounts/services-j4610176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-69f9b555fd-7q6cb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4610176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","signature_id":"75a9926b17ec082959d8c07a3aca44e4b153424b74c91ed1f5a5f6cbaf33d33f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106064405874610176","occurred_at":"2026-10-02T16:50:57Z","signature_id":"c2fcc826cd885e26911af5d73a5995409a57254fd3df2439b6894c3199cddb54","pr_number":7309,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.0303030303030303,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-27T19:23:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00378d080\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp.j6618240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"name\": \"19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:11:48.464360183Z\",\n \"endTime\": \"2026-10-02T21:30:51.922878069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00378d080\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp.j6618240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"name\": \"19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:11:48.464360183Z\",\n \"endTime\": \"2026-10-02T21:30:51.922878069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00063e600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-snc8bz4l2lpm\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"name\": \"377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:41:21.075539741Z\",\n \"endTime\": \"2026-10-02T12:00:23.993693871Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00063e600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-snc8bz4l2lpm\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"name\": \"377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:41:21.075539741Z\",\n \"endTime\": \"2026-10-02T12:00:23.993693871Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109176676618240","occurred_at":"2026-10-02T19:48:51Z","signature_id":"0b09878cead7ed0db7149a6d094f72740b9f49ac8eb39886c52d3d3e4afc2dc1","pr_number":7326,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"8dbea7b9ec3df592e7098abc9c237a0026e2477ed890812177c981f0246ac466","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.0303030303030303,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=1)","occurrences":2}],"full_error_samples":["time=2026-10-02T04:18:24.964Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-02T04:18:25.604Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 values=\"map[azure:map[clientId:3a6b5dce-c503-483f-868b-3656736645ed cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j6706176.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j6706176-mgmt-1 credsKeyVault:map[name:ah-ci01-mg-j6706176-1 secret:hcp-underlay-j6706176-mgmt-1] deployment:map[zoneCount:3] glog_v:2 image:map[digest:sha256:9af415e086251e94f471e03844ce768bed6493653b6cabdaff68acf99573679d registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:3a8316d42c6491dbc15f2cf59985901365c7c402d3a0649abe695a4a6554654e registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-10-02T04:18:25.604Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:18:26.032Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 namespace=maestro\ntime=2026-10-02T04:18:26.347Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-02T04:18:26.347Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:18:26.470Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:23:32.794Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-02T04:23:32.795Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-02T04:04:12.931Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-02T04:04:13.290Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 values=\"map[azure:map[clientId:ad930020-a0d3-4a84-b682-06665523a404 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j3304448.canadacentral-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j3304448-mgmt-1 credsKeyVault:map[name:ah-ci01-mg-j3304448-1 secret:hcp-underlay-j3304448-mgmt-1] deployment:map[zoneCount:3] glog_v:2 image:map[digest:sha256:9af415e086251e94f471e03844ce768bed6493653b6cabdaff68acf99573679d registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:3a8316d42c6491dbc15f2cf59985901365c7c402d3a0649abe695a4a6554654e registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-10-02T04:04:13.290Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:04:13.437Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 namespace=maestro\ntime=2026-10-02T04:04:13.594Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-02T04:04:13.594Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:04:13.638Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:09:17.935Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-02T04:09:17.936Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105865001846706176","occurred_at":"2026-10-02T03:38:36Z","signature_id":"65d9370e60af642e906378ad8145d81c4e331f8977d3894f60b0c564c92ed6e5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860601363304448","occurred_at":"2026-10-02T03:21:06Z","signature_id":"4ab9e4e3b3cde7399dc6f8c4bc34053ae4761a23ab44459a95beb2590036596e","pr_number":7105,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":18,"prior_last_seen_at":"2026-09-25T20:36:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-c74lpp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001776360\u003e: \n DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-c74lpp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001776360\u003e: \n DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","signature_id":"a9519ed127543f85f80a4754e183aabc6e52e7d298e88306d50cff9591c5662c","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,8,1,19,3,5,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gc94n755ktzv/cluster-candidate-5-0-l6z5kw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001032d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-l6z5kw: failed waiting for cluster=\"cluster-candidate-5-0-l6z5kw\" in resourcegroup=\"rg-candidate-5-0-gc94n755ktzv\" to finish creating: GET https://rp.j1697920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"name\": \"af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:49:05.164042474Z\",\n \"endTime\": \"2026-10-02T12:18:11.434067357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gc94n755ktzv/cluster-candidate-5-0-l6z5kw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001032d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-l6z5kw: failed waiting for cluster=\"cluster-candidate-5-0-l6z5kw\" in resourcegroup=\"rg-candidate-5-0-gc94n755ktzv\" to finish creating: GET https://rp.j1697920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"name\": \"af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:49:05.164042474Z\",\n \"endTime\": \"2026-10-02T12:18:11.434067357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","signature_id":"6151f3dfe8d1d7e6a99338873294fafd7a249d93b2444cb2b6df005a60d29dcc","pr_number":7303,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T16:19:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca0a60\u003e: \n failed to create HCP cluster ea-listpkp2p7: failed waiting for cluster=\"ea-listpkp2p7\" in resourcegroup=\"ea-list-4ccm8pbgrf47\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"name\": \"ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:58:05.107845034Z\",\n \"endTime\": \"2026-10-02T16:27:08.62997793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 1m14s ago; version 4.21.36 is Partial (want Completed), started 5m36s ago; version 4.21.36 is Partial (want Completed), started 11m4s ago; version 4.21.36 is Partial (want Completed), started 15m6s ago; version 4.21.36 is Partial (want Completed), started 20m46s ago; version 4.21.36 is Partial (want Completed), started 24m30s ago; version 4.21.36 is Partial (want Completed), started 25m39s ago; version 4.21.36 is Partial (want Completed), started 25m51s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca0a60\u003e: \n failed to create HCP cluster ea-listpkp2p7: failed waiting for cluster=\"ea-listpkp2p7\" in resourcegroup=\"ea-list-4ccm8pbgrf47\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"name\": \"ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:58:05.107845034Z\",\n \"endTime\": \"2026-10-02T16:27:08.62997793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 1m14s ago; version 4.21.36 is Partial (want Completed), started 5m36s ago; version 4.21.36 is Partial (want Completed), started 11m4s ago; version 4.21.36 is Partial (want Completed), started 15m6s ago; version 4.21.36 is Partial (want Completed), started 20m46s ago; version 4.21.36 is Partial (want Completed), started 24m30s ago; version 4.21.36 is Partial (want Completed), started 25m39s ago; version 4.21.36 is Partial (want Completed), started 25m51s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","signature_id":"6bce40273864c881050d6e2d7a4a0595d348985d0aa9ebd379d5a02487091686","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc37414a0bb07deceea16e285252404618c039575a3001bedcbcbd958dd8e87d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Monitor/accounts/services-j7538560; provider Microsoft.Monitor","search_query":"\"error\": {","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to perform alert processing rule CRUD operations","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to perform alert processing rule CRUD operations","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:493]: failed to create alert processing rule\nUnexpected error:\n \u003c*errors.errorString | 0xc0013d1ed0\u003e: \n expected status 200, got 400: {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/services-j7538560\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:493]: failed to create alert processing rule\nUnexpected error:\n \u003c*errors.errorString | 0xc0013d1ed0\u003e: \n expected status 200, got 400: {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/services-j7538560\"\n }\n }\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106164805017538560","occurred_at":"2026-10-02T23:29:54Z","signature_id":"1d1306e065199e57cf378e8b1110f97b0d8741e76dab448d010dc8c6539462a1","pr_number":7310,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,1,0,1,3,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-27T17:03:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x152c481045e0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s55djgjncgbr\" to finish creating: GET https://rp.j5664384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x152c481045e0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s55djgjncgbr\" to finish creating: GET https://rp.j5664384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106171709915664384","occurred_at":"2026-10-02T23:57:21Z","signature_id":"0c35eab3a95f3ff5276180583f68218d788a369962226e679eb22ef600b7e783","pr_number":7326,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d8120\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d8120\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"61300846ff2ab19ece6d91bcf7c616bc6175f4c4f2bfefccd840a12b6302252a","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d75e828705d5e967c897d0839b4327d675cfc294e87dead6ef776571e28083d","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013d63c0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013d63c0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"1c150067c2543dc56eca317c3be63944fbdc62dfd7d1e1aae30701f1f3c48892","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-21T21:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:193]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c66c0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:193]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c66c0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"d285c70301184b9a72cbfbcf497f8bb394dd1683ff0d5b4206fc2a65203f44fb","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bcb79b4f02a40cf142723d935e5b38797b059e620022fb96857e13363ac752f3","failure_pattern":"UpdateHCPCluster20261001 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:369]: failed to update containerRegistry via PATCH\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fa8690\u003e: \n failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:369]: failed to update containerRegistry via PATCH\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fa8690\u003e: \n failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"b721d7faa183dc16fbcbacdfc908d1c3712e92ed9f08208bc9d61279db4bba79","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-25T19:17:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00156e168\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00156e168\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106002969722884096","occurred_at":"2026-10-02T12:46:50Z","signature_id":"0d41539b045a3d8d46974b35f8490c28db0d1f05714109888ade97dba1eae56d","pr_number":7321,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":15,"trend_counts":[1,0,5,3,4,5,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":46,"prior_last_seen_at":"2026-09-26T19:18:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:24:46Z\n Ended: 2026-10-02T12:39:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","signature_id":"85419bd9eb6fb528fe3ba7717310c0fef873348ff3a1d1a575a7c479071f9143","pr_number":7303,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[1,0,1,2,3,5,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":27,"prior_last_seen_at":"2026-09-26T22:54:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:11:46Z\n Ended: 2026-10-02T12:41:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3ec768d5-6cf8-4b9c-a960-342894bf7da7\"\n Description: Pod ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-02T12:11:46Z\n Ended: 2026-10-02T12:41:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3ec768d5-6cf8-4b9c-a960-342894bf7da7\"\n Description: Pod ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","signature_id":"acd915c3dfae3e54fbda06bf63f8d1c3e877fc8e6f609f4e70e5b6899c2c7a85","pr_number":7303,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":20,"trend_counts":[1,1,1,5,2,5,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":44,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:19:46Z\n Ended: 2026-10-02T12:37:47Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T12:19:46Z\n Ended: 2026-10-02T12:37:46Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","signature_id":"ef955f75a2acdcd9710f3aa77383bb014db2593d410f1681e803db0fd48a8c18","pr_number":7303,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":20,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":38,"prior_last_seen_at":"2026-09-25T08:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:19:29Z\n Ended: 2026-10-03T00:25:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9271680-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9271680-svc).\nFiring 2:\n State: Resolved\n Started: 2026-10-03T00:35:30Z\n Ended: 2026-10-03T00:41:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9271680-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9271680-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7330/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157312019271680","occurred_at":"2026-10-02T23:00:08Z","signature_id":"46d148353c0b164c86fe30292f88ae89ac37e04f171a8c6e854febc1e90da782","pr_number":7330,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,2,0,0,0,1,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":20,"prior_last_seen_at":"2026-09-27T12:48:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002838820\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002838820\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","signature_id":"dfa343b19e8ca6f894f17713a054300c8d7a7c712659fa089dac380e5a5e1ab0","pr_number":7311,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:212]: Timed out after 2700.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:210 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: ...\nto be \u003c\n \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:212]: Timed out after 2700.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:210 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: ...\nto be \u003c\n \u003cint\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105987324146880512","occurred_at":"2026-10-02T11:44:40Z","signature_id":"34274ed735384eaaa0e2b1ae2d0acce3ecf44d677f61c5350015ce1f88a3aa46","pr_number":6807,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"17e5892e8c32879055400ab93a946ce79ad6cc684588825e1e1a488695d06f44","failure_pattern":"failed to poll cluster \"\u003ccluster\u003e\" authorized CIDRs update to completion (timeout '10.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:416]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:416]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e0fae1df9008b1d12ccc10847571ae065f9b820f22326abb57a797bc01d29b98","failure_pattern":"failed to update cluster \"\u003ccluster\u003e\" authorized CIDRs to include test runner IP","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","signature_id":"da4948d61ab54e112557ad6fe48bbeb3135dc01594a49200da12544749a36fa1","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.5151515151515151,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[1,0,0,2,0,0,1],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-26T16:16:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105830212750020608","occurred_at":"2026-10-02T01:20:21Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":7293,"after_last_push_of_merged_pr":false}]}]}]}